diff --git a/.github/workflows/test_pytest.yml b/.github/workflows/test_pytest.yml index 6ef157b3f..8e7af391e 100644 --- a/.github/workflows/test_pytest.yml +++ b/.github/workflows/test_pytest.yml @@ -129,5 +129,5 @@ jobs: - name: Run ci_engine session run: | cd contrib - python ci_engine.py -n 5 + python ci_engine.py run -n 5 timeout-minutes: 5 diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index bbbac8e4c..7fc2d19a0 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -41,18 +41,26 @@ """ - +import argparse +import asyncio +import collections +from concurrent.futures import ThreadPoolExecutor import configparser import itertools import json import logging +import operator +from pathlib import Path +import shlex +import signal import sqlite3 -import subprocess import sys +import threading +from tempfile import TemporaryDirectory from random import Random -import click from rich.console import Console +from rich.progress import (Progress, SpinnerColumn, TextColumn, TimeElapsedColumn) from rich.table import Table from pelita.network import RemotePlayerFailure @@ -64,114 +72,177 @@ # the path of the configuration file CFG_FILE = './ci.cfg' -def hash_team(team_spec): +EXIT = threading.Event() + +def signal_handler(_signal, _frame): + _logger.warning('Program terminated by kill or ctrl-c') + EXIT.set() + sys.exit() + +signal.signal(signal.SIGINT, signal_handler) + +async def hash_team(team_spec, semaphore): external_call = [sys.executable, '-m', 'pelita.scripts.pelita_player', 'hash-team', team_spec] - _logger.debug("Executing: %r", external_call) - res = subprocess.run(external_call, capture_output=True, text=True) - return res.stdout.strip().split("\n")[-1].strip() + async with semaphore: + _logger.debug("Executing: %r", shlex.join(external_call)) + proc = await asyncio.create_subprocess_exec(*external_call, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await proc.communicate() + + return stdout.decode().strip().split("\n")[-1].strip() + + +def run_game(team_specs, config): + """Run a single game. + + This method runs a single game and returns the result. + + Parameters + ---------- + p1, p2 : int + the indices of the players + + """ + + with TemporaryDirectory() as tmpdir: + final_state, stdout, stderr = call_pelita(team_specs, + rounds=config['rounds'], + size=config['size'], + viewer=config['viewer'], + seed=config['seed'], + store_output=tmpdir, + timeout=10, + initial_timeout=120, + exit_flag=EXIT + ) + + if not final_state: + result = None + + if final_state['game_phase'] != 'FINISHED': + _logger.info("Game finished in phase %s", final_state['game_phase']) + result = -2 + else: + if final_state['whowins'] == 2: + result = -1 + else: + result = final_state['whowins'] + + try: + del final_state['walls'] + del final_state['food'] + except IndexError: + pass + + _logger.info('Final state: %r', final_state) + _logger.debug('Stdout: %r', stdout) + if stderr: + _logger.warning('Stderr: %r', stderr) + + p1_stdout = (Path(tmpdir) / 'blue.out').read_text() + p1_stderr = (Path(tmpdir) / 'blue.err').read_text() + + p2_stdout = (Path(tmpdir) / 'red.out').read_text() + p2_stderr = (Path(tmpdir) / 'red.err').read_text() + + res = (result, final_state, [stdout, stderr], [p1_stdout, p1_stderr], [p2_stdout, p2_stderr]) + return res + + class CI_Engine: """Continuous Integration Engine.""" - def __init__(self, cfgfile): - self.players = [] + def __init__(self, cfgfile, database=None): + self.cfg_path = Path(cfgfile) + + self.players = {} config = configparser.ConfigParser() - config.read_file(cfgfile) + config.read(cfgfile) for name, path in config.items('agents'): - self.players.append({'name': name, 'path': path}) + self.players[name]= {'path': str(self.cfg_path.parent / path)} + self.rounds = config['general'].getint('rounds', None) self.size = config['general'].get('size', None) self.viewer = config['general'].get('viewer', 'null') self.seed = config['general'].get('seed', None) - self.db_file = config.get('general', 'db_file') + self.db_file = database or config.get('general', 'db_file') self.dbwrapper = DB_Wrapper(self.db_file) - def load_players(self): + def load_players(self, concurrency=1): hash_cache = {} # remove players from db which are not in the config anymore for pname in self.dbwrapper.get_players(): - if pname not in [p['name'] for p in self.players]: + if pname not in self.players: _logger.debug('Removing %s from database, because it is not among the current players.' % (pname)) self.dbwrapper.remove_player(pname) + semaphore = asyncio.Semaphore(concurrency) + + async def do_hash(): + players = [(pname, player['path']) for pname, player in self.players.items()] + tasks = [asyncio.create_task(hash_team(player[1], semaphore)) for player in players] + hashes = await asyncio.gather(*tasks) + return {player[0]: hash for (player, hash) in zip(players, hashes)} + + hash_cache = asyncio.run(do_hash()) + # add new players into db - for player in self.players: - pname, path = player['name'], player['path'] + for pname, player in self.players.items(): + path = player['path'] if pname not in self.dbwrapper.get_players(): _logger.debug('Adding %s to database.' % pname) - hash_cache[path] = hash_team(path) - self.dbwrapper.add_player(pname, hash_cache[path]) + self.dbwrapper.add_player(pname, hash_cache[pname]) # reset players where the directory hash changed - for player in self.players: + for pname, player in self.players.items(): path = player['path'] - pname = player['name'] - new_hash = hash_cache.get(path, hash_team(path)) + new_hash = hash_cache[pname] if new_hash != self.dbwrapper.get_player_hash(pname): _logger.debug('Resetting %s because its module hash changed.' % pname) self.dbwrapper.remove_player(pname) self.dbwrapper.add_player(pname, new_hash) - for player in self.players: - path = player['path'] - pname = player['name'] + def check_team_name(args): + pname, path = args try: _logger.debug('Querying team name for %s.' % pname) - team_name = check_team(player['path']) - self.dbwrapper.add_team_name(pname, team_name) + team_name = check_team(path, timeout=6*concurrency) + return { 'team_name': team_name } except RemotePlayerFailure as e: e_type, e_msg = e.args - _logger.debug(f'Could not import {player} at path {path} ({e_type}): {e_msg}') - player['error'] = e.args - - def run_game(self, p1, p2): - """Run a single game. - - This method runs a single game ``p1`` vs ``p2`` and internally - stores the result. - - Parameters - ---------- - p1, p2 : int - the indices of the players + _logger.debug(f'Could not import {pname} at path {path} ({e_type}): {e_msg}') + return { 'error': e.args } - """ - team_specs = [self.players[i]['path'] for i in (p1, p2)] - print(f"Playing {self.players[p1]['name']} against {self.players[p2]['name']}.") + with ThreadPoolExecutor(max_workers=concurrency) as executor: + players = [(pname, player['path']) for pname, player in self.players.items()] + team_names = executor.map(check_team_name, players) - final_state, stdout, stderr = call_pelita(team_specs, - rounds=self.rounds, - size=self.size, - viewer=self.viewer, - seed=self.seed) - - if final_state['whowins'] == 2: - result = -1 - else: - result = final_state['whowins'] - - del final_state['walls'] - del final_state['food'] - - _logger.info('Final state: %r', final_state) - _logger.debug('Stdout: %r', stdout) - if stderr: - _logger.warning('Stderr: %r', stderr) - p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] - self.dbwrapper.add_gameresult(p1_name, p2_name, result, final_state, stdout, stderr) + for (pname, path), team_name in zip(players, team_names): + if 'error' in team_name: + self.players[pname]['error'] = team_name['error'] + else: + self.dbwrapper.add_team_name(pname, team_name['team_name']) + for pname in self.players: + if 'error' in self.players[pname]: + print(pname, self.players[pname]) + else: + print(pname, self.players[pname], self.dbwrapper.get_team_name(pname)) - def start(self, n): + def start(self, n, concurrency): """Start the Engine. - This method will start and infinite loop, testing each agent + This method will start and run n matches, testing each agent randomly against another one. The result is printed after each game. @@ -183,27 +254,91 @@ def start(self, n): >>> ci.start() """ - loop = itertools.repeat(None) if n == 0 else itertools.repeat(None, n) - rng = Random() - - for _ in loop: - # choose the player with the least number of played game, - # match with another random player - # mix the sides and let them play - broken_players = {idx for idx, player in enumerate(self.players) if player.get('error')} - game_count = [(self.dbwrapper.get_game_count(p['name']), idx) for idx, p in enumerate(self.players)] - players_sorted = [idx for count, idx in sorted(game_count) if idx not in broken_players] - a, rest = players_sorted[0], players_sorted[1:] - b = rng.choice(rest) - players = [a, b] - rng.shuffle(players) - - self.run_game(players[0], players[1]) - self.pretty_print_results(highlight=[self.players[players[0]]['name'], self.players[players[1]]['name']]) - print('------------------------------') - - - def get_results(self, idx, idx2=None): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + TimeElapsedColumn() + ) as progress: + + lock = threading.Lock() + + def worker(count, p1, p2): + with lock: + progress_task = progress.add_task(f"Playing #{count}: {p1} against {p2}.") + + config = { + 'rounds': self.rounds, + 'size': self.size, + 'viewer': self.viewer, + 'seed': None, # TODO + } + + team_specs = [self.players[p1]['path'], self.players[p2]['path']] + res = run_game(team_specs, config) + + with lock: + progress.update(progress_task, completed=True, visible=False) + + return count, (p1, p2), res + + def producer(): + rng = Random() + + game_counts = self.dbwrapper.get_game_counts() + + for count in range(n): + for pname, player in self.players.items(): + if "error" in player and pname in game_counts: + del game_counts[pname] + + # choose the player with the least number of played games, + # match with another random player + # shuffle the sides and let them play + + players_sorted = sorted(list(game_counts.items()), key=operator.itemgetter(1)) + + a = players_sorted[0][0] + b = rng.choice(players_sorted[1:])[0] + + players = [a, b] + rng.shuffle(players) + + _logger.debug(f"Adding match {count} ({players[0]} vs {players[1]}) to worker queue") + task = (count, players[0], players[1]) + + yield task + + game_counts[a] += 1 + game_counts[b] += 1 + + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + if sys.version_info < (3, 14): + _logger.warning(f"Generating all {n} match partners. Use Python 3.14+ to do this lazily.") + buffersize = {} + else: + buffersize = {'buffersize': concurrency} + + for result in executor.map(lambda args: worker(*args), producer(), **buffersize): + count, players, res = result + + p1_name, p2_name = players + winner, final_state, out, p1_out, p2_out = res + + if final_state: + match final_state["whowins"]: + case 0: + progress.console.print(f"Storing #{count}: [u]{players[0]}[/u] against {players[1]}.") + case 1: + progress.console.print(f"Storing #{count}: {players[0]} against [u]{players[1]}[/u].") + case _: + progress.console.print(f"Storing #{count}: {players[0]} against {players[1]}.") + else: + progress.console.print(f"Not storing #{count}: {players[0]} against {players[1]}.") + self.dbwrapper.add_gameresult(p1_name, p2_name, winner, final_state, out, p1_out, p2_out) + + + def get_results(self, p1_name, p2_name=None): """Get the results so far. This method goes through the internal list of of all game @@ -230,7 +365,6 @@ def get_results(self, idx, idx2=None): the number of wins, losses and draws for this player or combination of players - Examples -------- @@ -245,18 +379,16 @@ def get_results(self, idx, idx2=None): """ win, loss, draw = 0, 0, 0 - p1_name = self.players[idx]['name'] - p2_name = None if idx2 is None else self.players[idx2]['name'] relevant_results = self.dbwrapper.get_results(p1_name, p2_name) for p1, p2, r in relevant_results: - if (idx2 is None and p1_name == p1) or (idx2 is not None and p1_name == p1 and p2_name == p2): + if (p2_name is None and p1_name == p1) or (p2_name is not None and p1_name == p1 and p2_name == p2): if r == 0: win += 1 elif r == 1: loss += 1 elif r == -1: draw += 1 - if (idx2 is None and p1_name == p2) or (idx2 is not None and p1_name == p2 and p2_name == p1): + if (p2_name is None and p1_name == p2) or (p2_name is not None and p1_name == p2 and p2_name == p1): if r == 1: win += 1 elif r == 0: @@ -265,7 +397,7 @@ def get_results(self, idx, idx2=None): draw += 1 return win, loss, draw - def get_errorcount(self, idx): + def get_errorcount(self, p_name): """Gets the error count for team idx Parameters @@ -279,17 +411,15 @@ def get_errorcount(self, idx): the number of errors for this player """ - p_name = self.players[idx]['name'] error_count, fatalerror_count = self.dbwrapper.get_errorcount(p_name) return error_count, fatalerror_count - def get_team_name(self, idx): + def get_team_name(self, p_name): """Get last registered team name. team_name : string """ - p_name = self.players[idx]['name'] return self.dbwrapper.get_team_name(p_name) def gen_elo(self): @@ -300,104 +430,37 @@ def elo_change(a, b, outcome): return k * (outcome - expected) from collections import defaultdict - elo = defaultdict(lambda: 1500) + elo = defaultdict(lambda: 1500.) g = self.dbwrapper.cursor.execute(""" SELECT player1, player2, result FROM games """).fetchall() for p1, p2, result in g: + change = 0 if result == 0: change = elo_change(elo[p1], elo[p2], 1) if result == 1: change = elo_change(elo[p1], elo[p2], 0) if result == -1: change = elo_change(elo[p1], elo[p2], 0.5) + elo[p1] += change elo[p2] -= change return elo - def pretty_print_results(self, highlight=None): + def pretty_print_results(self, full=False, team=None, highlight=None, html_export=None): """Pretty print the current results. """ if highlight is None: highlight = [] - console = Console() - # Some guesswork in here - MAX_COLUMNS = (console.width - 40) // 12 - if MAX_COLUMNS < 4: - # Let’s be honest: You should enlarge your terminal window even before that - MAX_COLUMNS = 4 - - res = self.dbwrapper.get_wins_losses() - rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } - - good_players = [p for p in self.players if not p.get('error')] - bad_players = [p for p in self.players if p.get('error')] - - num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 - row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] - - table = Table(row_styles=row_style, title="Cross results") - table.add_column("") - table.add_column("Name") - table.add_column("Score", justify="right") - table.add_column("W/D/L") - - column_players = [[] for _idx in range(min(MAX_COLUMNS, len(good_players)))] - # if we have more good_players than allowed columns, we must wrap around - for idx, _p in enumerate(good_players): - column_players[idx % MAX_COLUMNS].append(idx) - - for midx in column_players: - table.add_column('\n'.join(map(str, midx))) - - - def batched(iterable, n): - # Backport from Python 3.12 - # batched('ABCDEFG', 3) → ABC DEF G - if n < 1: - raise ValueError('n must be at least one') - iterator = iter(iterable) - while batch := tuple(itertools.islice(iterator, n)): - yield batch - - result = [] - for idx, p in enumerate(good_players): - win, loss, draw = self.get_results(idx) - error_count, fatalerror_count = self.get_errorcount(idx) - try: - team_name = self.get_team_name(idx) - except ValueError: - team_name = None - score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) - result.append([score, win, draw, loss, p['name'], team_name, error_count, fatalerror_count]) - wdl = f"{win:3d},{draw:3d},{loss:3d}" - - try: - row = rows[p['name']] - except KeyError: - continue - vals = { k: (w,l,d) for _p1, k, w, l, d in row } - - cross_results = [] - for idx2, p2 in enumerate(good_players): - win, loss, draw = vals.get(p2['name'], (0, 0, 0)) - if idx == idx2: - cross_results.append(" - - - ") - else: - cross_results.append(f"{win:2d},{draw:2d},{loss:2d}") + good_players = [p for p, player in self.players.items() if not player.get('error')] + bad_players = [p for p, player in self.players.items() if player.get('error')] - for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): - if c == 0: - table.add_row(f"{idx}", p['name'], f"{score:.2f}", wdl, *r) - else: - table.add_row("", "", "", "", *r) - - console.print(table) + console = Console(record=True) table = Table(title="Bot ranking") @@ -408,10 +471,22 @@ def batched(iterable, n): table.add_column("# Losses") table.add_column("Score") table.add_column("ELO") - table.add_column("Error count") + table.add_column("# Timeouts") table.add_column("# Fatal Errors") - elo = self.gen_elo() + elo = dict(self.dbwrapper.get_elo()) + # elo = self.gen_elo() + + result = [] + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) result.sort(reverse=True) for [score, win, draw, loss, name, team_name, error_count, fatalerror_count] in result: @@ -424,7 +499,7 @@ def batched(iterable, n): f"{draw}", f"{loss}", f"{score:6.3f}", - f"{elo[name]: >4.0f}", + f"{elo.get(name, 0): >4.0f}", f"{error_count}", f"{fatalerror_count}", style=style, @@ -433,7 +508,124 @@ def batched(iterable, n): console.print(table) for p in bad_players: - print("% 30s ***%30s***" % (p['name'], p['error'])) + print("% 30s ***%30s***" % (p, self.players[p]['error'])) + + + if full: + # Some guesswork in here + MAX_COLUMNS = (console.width - 40) // 12 + if MAX_COLUMNS < 4: + # Let’s be honest: You should enlarge your terminal window even before that + MAX_COLUMNS = 4 + + res = self.dbwrapper.get_wins_losses() + rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } + + num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 + row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] + + table = Table(row_styles=row_style, title="Cross results") + table.add_column("") + table.add_column("Name") + table.add_column("Score", justify="right") + table.add_column("W/D/L") + + column_players = [[] for _idx in range(min(MAX_COLUMNS, len(good_players)))] + # if we have more good_players than allowed columns, we must wrap around + for idx, _p in enumerate(good_players): + column_players[idx % MAX_COLUMNS].append(idx) + + for midx in column_players: + table.add_column('\n'.join(map(str, midx))) + + + def batched(iterable, n): + # Backport from Python 3.12 + # batched('ABCDEFG', 3) → ABC DEF G + if n < 1: + raise ValueError('n must be at least one') + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + yield batch + + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + wdl = f"{win:3d},{draw:3d},{loss:3d}" + + try: + row = rows[pname] + except KeyError: + continue + vals = { k: (w,l,d) for _p1, k, w, l, d in row } + + cross_results = [] + for idx2, p2name in enumerate(good_players): + win, loss, draw = vals.get(p2name, (0, 0, 0)) + if idx == idx2: + cross_results.append(" - - - ") + else: + cross_results.append(f"{win:2d},{draw:2d},{loss:2d}") + + for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): + if c == 0: + table.add_row(f"{idx}", pname, f"{score:.2f}", wdl, *r) + else: + table.add_row("", "", "", "", *r) + + console.print(table) + + elif team: + MAX_COLUMNS = (console.width - 40) // 12 + if MAX_COLUMNS < 4: + # Let’s be honest: You should enlarge your terminal window even before that + MAX_COLUMNS = 4 + + res = self.dbwrapper.get_wins_losses(team=team) + rows = {k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[1])} + + row_style = ["", "dim"] + + table = Table(row_styles=row_style, title=f"Match results for team {team}") + table.add_column("Name") + table.add_column("# Matches") + table.add_column("# Wins") + table.add_column("# Draws") + table.add_column("# Losses") + + for idx, pname in enumerate(good_players): + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + + try: + row = rows[pname] + except KeyError: + continue + + for r in row: # there should only be one row + p1, p2, win, loss, draw = r + + display_name = f"{pname} ({team_name})" if team_name else f"{pname}" + + table.add_row( + display_name, + f"{win+draw+loss}", + f"{win}", + f"{draw}", + f"{loss}", + ) + + console.print(table) + + if html_export: + console.save_html(html_export) class DB_Wrapper: @@ -451,6 +643,7 @@ def __init__(self, dbfile): """ self.db_file = dbfile + _logger.info("Using sqlite database file ‘%s’.", self.db_file) self.connection = sqlite3.connect(self.db_file) self.cursor = self.connection.cursor() self.cursor.execute("PRAGMA foreign_keys = ON;") @@ -473,10 +666,22 @@ def create_tables(self): """) self.cursor.execute(""" CREATE TABLE IF NOT EXISTS games - (player1 text, player2 text, result int, final_state text, stdout text, stderr text, + ( + id INTEGER PRIMARY KEY, + player1 text, player2 text, result int, final_state text, + player1_num_timeouts int, player2_num_timeouts int, + player1_had_fatal_error bool, player2_had_fatal_error bool, FOREIGN KEY(player1) REFERENCES players(name) ON DELETE CASCADE, FOREIGN KEY(player2) REFERENCES players(name) ON DELETE CASCADE) """) + self.cursor.execute(""" + CREATE TABLE IF NOT EXISTS game_output + (game_id int, + stdout text, stderr text, + player1_stdout text, player1_stderr text, + player2_stdout text, player2_stderr text, + FOREIGN KEY(game_id) REFERENCES games(id) ON DELETE CASCADE) + """) self.connection.commit() def get_players(self): @@ -567,7 +772,7 @@ def remove_player(self, pname): WHERE name = ?""", (pname,)) self.connection.commit() - def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err): + def add_gameresult(self, p1_name, p2_name, result, final_state, std, p1_out, p2_out): """Add a new game result to the database. Parameters @@ -578,14 +783,43 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err 0 if player 1 won 1 of player 2 won -1 if draw + -2 if anything other than game_phase FINISHED std_out, std_err : str STDOUT and STDERR of the game """ + + stdout, stderr = std + p1_stdout, p1_stderr = p1_out + p2_stdout, p2_stderr = p2_out + + if not final_state: + return + + final_state_str = json.dumps(final_state) + player1_num_timeouts, player2_num_timeouts = final_state['num_timeouts'] + + player1_had_fatal_error = len(final_state['fatal_errors'][0]) != 0 + player2_had_fatal_error = len(final_state['fatal_errors'][1]) != 0 + self.cursor.execute(""" INSERT INTO games - VALUES (?, ?, ?, ?, ?, ?) - """, [p1_name, p2_name, result, json.dumps(final_state), std_out, std_err]) + (player1, player2, result, final_state, + player1_num_timeouts, player2_num_timeouts, + player1_had_fatal_error, player2_had_fatal_error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + RETURNING id + """, [p1_name, p2_name, result, final_state_str, + player1_num_timeouts, player2_num_timeouts, + player1_had_fatal_error, player2_had_fatal_error]) + + game_id, = self.cursor.fetchone() + self.cursor.execute(""" + INSERT INTO game_output + VALUES (?, ?, ?, ?, ?, ?, ?) + """, [game_id, + stdout, stderr, + p1_stdout, p1_stderr, p2_stdout, p2_stderr]) self.connection.commit() def get_results(self, p1_name, p2_name=None): @@ -640,6 +874,32 @@ def get_team_name(self, p_name): raise ValueError('Player %s does not exist in database.' % p_name) return res[0] + def get_game_counts(self): + """Get number of games per player. + + Returns + ------- + relevant_results : dict[name, int] + + """ + self.cursor.execute(""" + SELECT p.name, COUNT(g.player) AS num_games + FROM + players p + LEFT JOIN + ( + SELECT player1 AS player FROM games + UNION ALL + SELECT player2 AS player FROM games + ) g + ON p.name = g.player + GROUP BY p.name + """) + counts = collections.Counter() + for name, val in self.cursor.fetchall(): + counts[name] += val + return counts + def get_game_count(self, p1_name, p2_name=None): """Get number of games involving player1 (AND player2 if specified). @@ -681,45 +941,29 @@ def get_errorcount(self, p1_name): Returns ------- error_count, fatalerror_count : errorcount - """ self.cursor.execute(""" - SELECT sum(c) FROM + SELECT sum(timeouts), sum(fatal_errors) FROM ( - SELECT sum(json_extract(final_state, '$.num_errors[0]')) AS c + SELECT + sum(player1_num_timeouts) AS timeouts, + sum(player1_had_fatal_error) AS fatal_errors FROM games WHERE player1 = :p1 UNION ALL - SELECT sum(json_extract(final_state, '$.num_errors[1]')) AS c + SELECT + sum(player2_num_timeouts) AS timeouts, + sum(player2_had_fatal_error) AS fatal_errors FROM games WHERE player2 = :p1 ) """, dict(p1=p1_name)) - error_count, = self.cursor.fetchone() - - self.cursor.execute(""" - SELECT sum(c) FROM - ( - SELECT count(*) AS c - FROM games - WHERE player1 = :p1 AND - json_extract(final_state, '$.fatal_errors[0]') != '[]' - - UNION ALL - - SELECT count(*) AS c - FROM games - WHERE player2 = :p1 AND - json_extract(final_state, '$.fatal_errors[1]') != '[]' - ) - """, - dict(p1=p1_name)) - fatal_errorcount, = self.cursor.fetchone() + timeouts, fatal_errorcount = self.cursor.fetchone() - return error_count, fatal_errorcount + return timeouts, fatal_errorcount def get_wins_losses(self, team=None): """ Get all wins and losses combined in a table of @@ -800,31 +1044,130 @@ def get_wins_losses(self, team=None): return self.cursor.execute(query).fetchall() -@click.command() -@click.option('--log', - is_flag=False, flag_value="-", default=None, metavar='LOGFILE', - help="print debugging log information to LOGFILE (default 'stderr')") -@click.option('--config', - default=CFG_FILE, - type=click.File('r'), - help='Configuration file') -@click.option('-n', help='run N times', type=int, default=0) -@click.option('--print', is_flag=True, default=False, - help='Print scores and exit.') -@click.option('--nohash', is_flag=True, default=False, - help='Do not hash the players') -def main(log, config, n, print, nohash): - if log is not None: - start_logging(log, __name__) - start_logging(log, 'pelita') - - ci_engine = CI_Engine(config) - if print: - ci_engine.pretty_print_results() - else: - if not nohash: - ci_engine.load_players() - ci_engine.start(n) + def get_elo(self): + query = """ + WITH RECURSIVE + ordered_matches AS ( + SELECT + ROW_NUMBER() OVER (ORDER BY rowid) AS match_num, + player1, + player2, + result + FROM games + ), + + -- Initialize with first match + elo_recursive(match_num, player1, player2, result, + rating1, rating2, + rating_json) AS ( + SELECT + match_num, + player1, + player2, + result, + 1500.0, + 1500.0, + json_object(player1, 1500.0, player2, 1500.0) + FROM ordered_matches + WHERE match_num = 1 + + UNION ALL + + SELECT + om.match_num, + om.player1, + om.player2, + om.result, + + -- Get ratings from JSON state + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0), + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0), + + -- Update JSON state with new ratings + json_set( + er.rating_json, + '$.' || om.player1, + ROUND( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) + + 32 * ((CASE om.result WHEN 0 THEN 1.0 WHEN -1 THEN 0.5 ELSE 0.0 END) - + 1.0 / (1 + pow(10, ( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) - + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) + ) / 400.0))), 2), + '$.' || om.player2, + ROUND( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) + + 32 * ((CASE om.result WHEN 0 THEN 0.0 WHEN -1 THEN 0.5 ELSE 1.0 END) - + 1.0 / (1 + pow(10, ( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) - + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) + ) / 400.0))), 2) + ) + FROM ordered_matches om + JOIN elo_recursive er ON om.match_num = er.match_num + 1 + ), + + final AS ( + SELECT rating_json + FROM elo_recursive + ORDER BY match_num DESC + LIMIT 1 + ) + SELECT + key AS player, + ROUND(value, 2) AS rating + FROM final, json_each(rating_json) + ORDER BY rating DESC; + + """ + return self.cursor.execute(query).fetchall() + +def run(args): + ci_engine = CI_Engine(args.config, args.database) + if not args.no_hash: + ci_engine.load_players(concurrency=args.thread_count) + ci_engine.start(args.n, args.thread_count) + +def print_scores(args): + ci_engine = CI_Engine(args.config, args.database) + ci_engine.pretty_print_results(full=args.full, team=args.team, html_export=args.html_export) + +def hash_teams(args): + ci_engine = CI_Engine(args.config, args.database) + ci_engine.load_players(concurrency=args.thread_count) if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--log', help="Print debugging log information to LOGFILE (default 'stderr').", + metavar='LOGFILE', const='-', nargs='?') + parser.add_argument('--config', help="Print debugging log information to LOGFILE (default 'stderr').", + metavar='FILE', default=CFG_FILE) + parser.add_argument('--database', help="Database location", + metavar='FILE', default=None) + + subparsers = parser.add_subparsers(required=True) + + parser_run = subparsers.add_parser('run') + parser_run.add_argument('-n', help='run N times', type=int, default=1000) + parser_run.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) + parser_run.add_argument('--no-hash', help='Do not hash the players prior to running', action='store_true', default=False) + parser_run.set_defaults(func=run) + + parser_print_scores = subparsers.add_parser('print-scores') + parser_print_scores.add_argument('--html-export', help='output as HTML', default=False) + full_or_team = parser_print_scores.add_mutually_exclusive_group() + full_or_team.add_argument('--full', help='show full pair statistics', action='store_true', default=False) + full_or_team.add_argument('--team', help='show statistics for team', type=str, default=None) + parser_print_scores.set_defaults(func=print_scores) + + parser_hash = subparsers.add_parser('hash-teams') + parser_hash.set_defaults(func=hash_teams) + parser_hash.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) + + args = parser.parse_args() + + if args.log is not None: + start_logging(args.log, __name__) + start_logging(args.log, 'pelita') + + args.func(args) diff --git a/contrib/test_ci_engine.py b/contrib/test_ci_engine.py index 2e59e5218..cc3939d0f 100644 --- a/contrib/test_ci_engine.py +++ b/contrib/test_ci_engine.py @@ -8,6 +8,9 @@ def db_wrapper(): wrapper.create_tables() return wrapper +def make_simple_gameresult(p1, p2, result): + return [p1, p2, result, {'num_timeouts': [0, 0], 'fatal_errors': [[], []]}, ['', ''], ['', ''], ['', '']] + """Tests for the DB_Wrapper class.""" def test_foreign_keys_enabled(db_wrapper): @@ -26,9 +29,9 @@ def test_remove_player(db_wrapper): db_wrapper.add_player('p1', 'h1') db_wrapper.add_player('p2', 'h2') db_wrapper.add_player('p3', 'h3') - db_wrapper.add_gameresult('p1', 'p2', 0, '{}', '', '') - db_wrapper.add_gameresult('p2', 'p1', 0, '{}', '', '') - db_wrapper.add_gameresult('p2', 'p3', 0, '{}', '', '') + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', 0)) + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p1', 0)) + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p3', 0)) # player2 has three games assert len(db_wrapper.get_results('p2')) == 3 db_wrapper.remove_player('p1') @@ -63,13 +66,13 @@ def test_get_results(db_wrapper): db_wrapper.add_player('p2', 'h2') # empty list if no results are available assert db_wrapper.get_results('p1') == [] - db_wrapper.add_gameresult('p1', 'p2', 0, '{}', '', '') + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', 0)) result = db_wrapper.get_results('p1')[0] # check for correct values assert result[0] == 'p1' assert result[1] == 'p2' assert result[2] == 0 - db_wrapper.add_gameresult('p2', 'p1', 0, '{}', '', '') + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p1', 0)) # check for correct number of results results = db_wrapper.get_results('p1') assert len(results) == 2 @@ -97,22 +100,22 @@ def test_wins_losses(db_wrapper): db_wrapper.add_player('p1', 'h1') db_wrapper.add_player('p2', 'h2') db_wrapper.add_player('p3', 'h3') - db_wrapper.add_gameresult('p1', 'p2', 0, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', 0)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 1, 0, 0), ('p2', 'p1', 0, 1, 0) ] - db_wrapper.add_gameresult('p1', 'p2', -1, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', -1)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 1, 0, 1), ('p2', 'p1', 0, 1, 1) ] - db_wrapper.add_gameresult('p2', 'p1', 1, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p1', 1)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 2, 0, 1), ('p2', 'p1', 0, 2, 1) ] - db_wrapper.add_gameresult('p3', 'p1', 1, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p3', 'p1', 1)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 2, 0, 1), ('p1', 'p3', 1, 0, 0), @@ -137,3 +140,5 @@ def test_wins_losses(db_wrapper): assert db_wrapper.get_game_count('p1', 'p2') == 3 assert db_wrapper.get_game_count('p2', 'p1') == 3 assert db_wrapper.get_game_count('p3', 'p1') == 1 + + assert db_wrapper.get_game_counts() == dict(p1=4, p2=3, p3=1) diff --git a/pelita/game.py b/pelita/game.py index 5651ae884..29d0aac1a 100644 --- a/pelita/game.py +++ b/pelita/game.py @@ -3,6 +3,7 @@ import logging import math import os +import shlex import subprocess import sys import time @@ -96,7 +97,7 @@ def _run_external_viewer(self, subscribe_sock, controller, geometry, delay, stop external_call = [sys.executable, '-m', tkviewer] + viewer_args - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) # os.setsid will keep the viewer from closing when the main process exits # a better solution might be to decouple the viewer from the main process if _mswindows: @@ -804,7 +805,7 @@ def prepare_viewer_state(game_state): # game_state["timeouts"] has a tuple as a dict key # that cannot be serialized in json. # To fix this problem, we only send the current error - # and add another attribute "num_errors" + # and add another attribute "num_timeouts" # to the final dict. # the key for the current round, turn @@ -816,7 +817,7 @@ def prepare_viewer_state(game_state): ] # add the number of errors - viewer_state["num_errors"] = [ + viewer_state["num_timeouts"] = [ len(team_errors) for team_errors in game_state["timeouts"] ] diff --git a/pelita/scripts/pelita_server.py b/pelita/scripts/pelita_server.py index 3db613a1a..2591bc97d 100755 --- a/pelita/scripts/pelita_server.py +++ b/pelita/scripts/pelita_server.py @@ -506,7 +506,7 @@ def play_remote(team_spec, pair_addr, silent_bots=False): pair_addr, *(['--silent-bots'] if silent_bots else []), ] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) sub = subprocess.Popen(external_call) return sub @@ -517,7 +517,7 @@ def _check_team(team_spec): 'pelita.scripts.pelita_player', 'check-team', team_spec] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) res = subprocess.run(external_call, capture_output=True, text=True) return res.stdout.strip() diff --git a/pelita/scripts/pelita_tournament.py b/pelita/scripts/pelita_tournament.py index 7515561c1..e53635da5 100755 --- a/pelita/scripts/pelita_tournament.py +++ b/pelita/scripts/pelita_tournament.py @@ -37,29 +37,6 @@ def firstNN(*args): """ return next(filter(lambda x: x is not None, args), None) - -def shlex_unsplit(cmd): - """ - Translates a list of command arguments into bash-like ‘human’ readable form. - Pseudo-reverses shlex.split() - - Example - ------- - >>> shlex_unsplit(["command", "-f", "Hello World"]) - "command -f 'Hello World'" - - Parameters - ---------- - cmd : list of string - command + parameter list - - Returns - ------- - string - """ - return " ".join(shlex.quote(arg) for arg in cmd) - - def create_directory(prefix): for suffix in itertools.count(0): path = Path('{}-{:02d}'.format(prefix, suffix)) @@ -139,9 +116,9 @@ def setup(): print("Please enter the location of the sound-giving binary:") sound_path = input() elif res == "s": - sound_path = shlex_unsplit(sound["say"]) + sound_path = shlex.join(sound["say"]) elif res == "f": - sound_path = shlex_unsplit(sound["flite"]) + sound_path = shlex.join(sound["flite"]) else: continue diff --git a/pelita/team.py b/pelita/team.py index a8dee66f1..83e3d9a43 100644 --- a/pelita/team.py +++ b/pelita/team.py @@ -1,6 +1,7 @@ import logging import os +import shlex import subprocess import sys import time @@ -495,7 +496,7 @@ def _call_pelita_player(self, team_spec, address, color='', store_output=False): team_spec, address] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) if store_output == subprocess.DEVNULL: return (subprocess.Popen(external_call, stdout=store_output), None, None) elif store_output: diff --git a/pelita/tournament/__init__.py b/pelita/tournament/__init__.py index a2193edac..decd9a27c 100644 --- a/pelita/tournament/__init__.py +++ b/pelita/tournament/__init__.py @@ -98,7 +98,8 @@ def run_and_terminate_process(args, **kwargs): p.kill() -def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, write_replay=False, store_output=False): +def call_pelita(team_specs, *, rounds, size, viewer, seed, timeout=3, initial_timeout=6, + team_infos=None, write_replay=False, store_output=False, exit_flag=None): """ Starts a new process with the given command line arguments and waits until finished. Returns @@ -134,6 +135,8 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ size = ['--size', size] if size else [] viewer = ['--' + viewer] if viewer else [] seed = ['--seed', seed] if seed else [] + timeout = ['--timeout', str(timeout)] + initial_timeout = ['--initial-timeout', str(initial_timeout)] write_replay = ['--write-replay', write_replay] if write_replay else [] store_output = ['--store-output', store_output] if store_output else [] append_blue = ['--append-blue', team_infos[0]] if team_infos[0] else [] @@ -148,6 +151,8 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ *size, *viewer, *seed, + *timeout, + *initial_timeout, *write_replay, *store_output] @@ -167,6 +172,11 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ while True: evts = dict(poll.poll(1000)) + if exit_flag and exit_flag.is_set(): + # An external process tells us to quit + _logger.info("Received exit signal") + break + if not evts and proc.poll() is not None: # no more events and proc has finished. # we break the loop diff --git a/pelita/ui/tk_canvas.py b/pelita/ui/tk_canvas.py index 547e57ac8..a3993dc4b 100644 --- a/pelita/ui/tk_canvas.py +++ b/pelita/ui/tk_canvas.py @@ -870,7 +870,7 @@ def status(team_idx): # sum the deaths of both bots in this team deaths = game_state['deaths'][team_idx] + game_state['deaths'][team_idx+2] kills = game_state['kills'][team_idx] + game_state['kills'][team_idx+2] - ret = "Timeouts: %d, Kills: %d, Deaths: %d, Time: %.2f" % (game_state["num_errors"][team_idx], kills, deaths, game_state["team_time"][team_idx]) + ret = "Timeouts: %d, Kills: %d, Deaths: %d, Time: %.2f" % (game_state["num_timeouts"][team_idx], kills, deaths, game_state["team_time"][team_idx]) return ret except TypeError: return ""