Allow configuring subtasks to always show testcases.

Also defines a new (optional) format for parameters for ScoreTypeGroup.
Also allows specifying testcases by lists instead of by regex.

Author: veluca93

cms/grading/scoretypes/GroupThreshold.py

@@ -40,15 +40,21 @@ class GroupThreshold(ScoreTypeGroup):
 
     def get_public_outcome(self, outcome, parameter):
         """See ScoreTypeGroup."""
-        threshold = parameter[2]
+        if isinstance(parameter, list):
+            threshold = parameter[2]
+        else:
+            threshold = parameter["threshold"]
         if 0.0 < outcome <= threshold:
             return N_("Correct")
         else:
             return N_("Not correct")
 
     def reduce(self, outcomes, parameter):
         """See ScoreTypeGroup."""
-        threshold = parameter[2]
+        if isinstance(parameter, list):
+            threshold = parameter[2]
+        else:
+            threshold = parameter["threshold"]
         if all(0 < outcome <= threshold for outcome in outcomes):
             return 1.0
         else:

cms/grading/scoretypes/abc.py

@@ -32,6 +32,7 @@
 
 import logging
 import re
+from typing import TypedDict, NotRequired
 from abc import ABCMeta, abstractmethod
 
 from cms import FEEDBACK_LEVEL_RESTRICTED
@@ -192,26 +193,38 @@ class ScoreTypeAlone(ScoreType):
     pass
 
 
+class ScoreTypeGroupParametersDict(TypedDict):
+    max_score: float
+    testcases: int | str | list[str]
+    threshold: NotRequired[float]
+    always_show_testcases: NotRequired[bool]
+
+
+# the format of parameters is impossible to type-hint correctly, it seems...
+# this hint is (mostly) correct for the methods this base class implements,
+# subclasses might need a longer tuple.
+ScoreTypeGroupParameters = tuple[float, int | str | list[str]] | ScoreTypeGroupParametersDict
+
+
 class ScoreTypeGroup(ScoreTypeAlone):
     """Intermediate class to manage tasks whose testcases are
     subdivided in groups (or subtasks). The score type parameters must
     be in the form [[m, t, ...], [...], ...], where m is the maximum
     score for the given subtask and t is the parameter for specifying
-    testcases.
+    testcases, or be a list of dicts matching ScoreTypeGroupParametersDict.
 
     If t is int, it is interpreted as the number of testcases
     comprising the subtask (that are consumed from the first to the
     last, sorted by num). If t is unicode, it is interpreted as the regular
-    expression of the names of target testcases. All t must have the same type.
+    expression of the names of target testcases. If t is a list of strings,
+    it is interpreted as a list of testcase codenames. All t must have the
+    same type.
 
     A subclass must implement the method 'get_public_outcome' and
     'reduce'.
 
     """
-    # the format of parameters is impossible to type-hint correctly, it seems...
-    # this hint is (mostly) correct for the methods this base class implements,
-    # subclasses might need a longer tuple.
-    parameters: list[tuple[float, int | str]]
+    parameters: list[ScoreTypeGroupParameters]
 
     # Mark strings for localization.
     N_("Subtask %(index)s")
@@ -333,6 +346,24 @@ class ScoreTypeGroup(ScoreTypeAlone):
 </div>
 {% endfor %}"""
 
+    def get_max_score(self, group_parameter: ScoreTypeGroupParameters) -> float:
+        if isinstance(group_parameter, tuple) or isinstance(group_parameter, list):
+            return group_parameter[0]
+        else:
+            return group_parameter["max_score"]
+
+    def get_testcases(self, group_parameter: ScoreTypeGroupParameters) -> int | str | list[str]:
+        if isinstance(group_parameter, tuple) or isinstance(group_parameter, list):
+            return group_parameter[1]
+        else:
+            return group_parameter["testcases"]
+
+    def get_always_show_testcases(self, group_parameter: ScoreTypeGroupParameters) -> bool:
+        if isinstance(group_parameter, tuple) or isinstance(group_parameter, list):
+            return False
+        else:
+            return group_parameter.get("always_show_testcases", False)
+
     def retrieve_target_testcases(self) -> list[list[str]]:
         """Return the list of the target testcases for each subtask.
 
@@ -345,7 +376,7 @@ def retrieve_target_testcases(self) -> list[list[str]]:
 
         """
 
-        t_params = [p[1] for p in self.parameters]
+        t_params = [self.get_testcases(p) for p in self.parameters]
 
         if all(isinstance(t, int) for t in t_params):
 
@@ -379,9 +410,12 @@ def retrieve_target_testcases(self) -> list[list[str]]:
 
             return targets
 
+        elif all(isinstance(t, list) for t in t_params) and all(all(isinstance(t, str) for t in s) for s in t_params):
+            return t_params
+
         raise ValueError(
             "In the score type parameters, the second value of each element "
-            "must have the same type (int or unicode)")
+            "must have the same type (int, unicode or list of strings)")
 
     def max_scores(self):
         """See ScoreType.max_score."""
@@ -393,10 +427,10 @@ def max_scores(self):
 
         for st_idx, parameter in enumerate(self.parameters):
             target = targets[st_idx]
-            score += parameter[0]
+            score += self.get_max_score(parameter)
             if all(self.public_testcases[tc_idx] for tc_idx in target):
-                public_score += parameter[0]
-            headers += ["Subtask %d (%g)" % (st_idx, parameter[0])]
+                public_score += self.get_max_score(parameter)
+            headers += ["Subtask %d (%g)" % (st_idx, self.get_max_score(parameter))]
 
         return score, public_score, headers
 
@@ -461,10 +495,10 @@ def compute_score(self, submission_result):
             st_score_fraction = self.reduce(
                 [float(evaluations[tc_idx].outcome) for tc_idx in target],
                 parameter)
-            st_score = st_score_fraction * parameter[0]
+            st_score = st_score_fraction * self.get_max_score(parameter)
             rounded_score = round(st_score, score_precision)
 
-            if tc_first_lowest_idx is not None and st_score_fraction < 1.0:
+            if tc_first_lowest_idx is not None and st_score_fraction < 1.0 and not self.get_always_show_testcases(parameter):
                 for tc in testcases:
                     if not self.public_testcases[tc["idx"]]:
                         continue
@@ -482,7 +516,7 @@ def compute_score(self, submission_result):
                 "score_fraction": st_score_fraction,
                 # But we also want the properly rounded score for display.
                 "score": rounded_score,
-                "max_score": parameter[0],
+                "max_score": self.get_max_score(parameter),
                 "testcases": testcases})
             if all(self.public_testcases[tc_idx] for tc_idx in target):
                 public_score += st_score
@@ -495,7 +529,7 @@ def compute_score(self, submission_result):
         return score, subtasks, public_score, public_subtasks, ranking_details
 
     @abstractmethod
-    def get_public_outcome(self, outcome: float, parameter: list) -> str:
+    def get_public_outcome(self, outcome: float, parameter: ScoreTypeGroupParameters) -> str:
         """Return a public outcome from an outcome.
 
         The public outcome is shown to the user, and this method
@@ -512,7 +546,7 @@ def get_public_outcome(self, outcome: float, parameter: list) -> str:
         pass
 
     @abstractmethod
-    def reduce(self, outcomes: list[float], parameter: list) -> float:
+    def reduce(self, outcomes: list[float], parameter: ScoreTypeGroupParameters) -> float:
         """Return the score of a subtask given the outcomes.
 
         outcomes: the outcomes of the submission in