Switched to pytest style output

Author: wenting-zhao

commit0/harness/constants.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from pathlib import Path
-from typing import Dict, ItemsView
+from typing import Dict, ItemsView, KeysView
 from pydantic import BaseModel
 
 
@@ -16,17 +16,24 @@ class RepoInstance(BaseModel):
     def __getitem__(self, item: str):
         return getattr(self, item)
 
+    def keys(self) -> KeysView[str]:
+        """Return the field names of the model as dictionary keys."""
+        return self.__annotations__.keys()
+
 
 class SimpleInstance(BaseModel):
     instance_id: str
     prompt: str
     canonical_solution: str
     test: str
-    entry_point: str
 
     def __getitem__(self, item: str):
         return getattr(self, item)
 
+    def keys(self) -> KeysView[str]:
+        """Return the field names of the model as dictionary keys."""
+        return self.__annotations__.keys()
+
 
 class Files(BaseModel):
     eval_script: Dict[str, Path]

commit0/harness/run_pytest_ids.py

@@ -1,6 +1,5 @@
 import git
 import os
-import re
 import sys
 import traceback
 from datasets import load_dataset
@@ -21,6 +20,7 @@
     generate_patch_between_commits,
     setup_logger,
     close_logger,
+    extract_code_blocks,
 )
 from commit0.harness.execution_context import (
     ExecutionBackend,
@@ -165,15 +165,13 @@ def main(
             )
         else:
             solution = open(test_ids).read()
-            pattern = r"```python\n(.*?)```"
-            matches = re.finditer(pattern, solution, re.DOTALL)
-            matches = [match.group(1).strip() for match in matches]
+            prompt = example["prompt"] if "prompt" in example.keys() else ""
+            matches = extract_code_blocks(solution)
             if len(matches) > 0:
                 solution = "\n\n".join(matches)
             else:
-                solution = example["prompt"] + "\n\n" + solution
+                solution = prompt + "\n\n" + solution
             patch = solution + "\n\n" + example["test"]
-        patch = patch + "\n\n" + f"check({example['entry_point']})"
         eval_script = spec.eval_script
 
     patch_file = Path(log_dir / "patch.diff")

commit0/harness/spec.py

@@ -185,6 +185,7 @@ def make_repo_script_list(self) -> list[str]:
             f"mkdir {self.repo_directory} && cd {self.repo_directory}",
             "uv venv --python 3.12",
             "source .venv/bin/activate",
+            "uv pip install -U pytest pytest-cov coverage pytest-json-report",
             "which python",
         ]
         return setup_commands
@@ -195,7 +196,7 @@ def make_eval_script_list(self) -> list[str]:
             f"cd {self.repo_directory}",
             "source .venv/bin/activate",
             "cat /patch.diff > test.py",
-            "uv run test.py > test_output.txt 2>&1",
+            "pytest test.py > test_output.txt 2>&1",
             "echo $? > pytest_exit_code.txt",
         ]
         return eval_script_list

commit0/harness/utils.py

@@ -4,9 +4,10 @@
 import logging
 import os
 import time
+import re
 import sys
 from pathlib import Path
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from fastcore.net import HTTP404NotFoundError, HTTP403ForbiddenError  # type: ignore
 from ghapi.core import GhApi
@@ -220,4 +221,27 @@ def get_active_branch(repo_path: Union[str, Path]) -> str:
     return branch
 
 
+def extract_code_blocks(text: str) -> List[str]:
+    """Extract Python code blocks from a given text wrapped in markdown markers.
+
+    This function identifies and extracts all Python code blocks within a provided
+    text. The code blocks should be surrounded by markdown-style markers, such as
+    ```python ... ```.
+
+    Args:
+    ----
+        text (str): The input text containing Python code blocks marked with
+                    ```python ... ```.
+
+    Returns:
+    -------
+        List[str]: A list of strings, each containing a Python code block extracted
+                   from the text.
+
+    """
+    pattern = r"```python\n(.*?)```"
+    matches = re.finditer(pattern, text, re.DOTALL)
+    return [match.group(1).strip() for match in matches]
+
+
 __all__ = []