diff --git a/tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py new file mode 100644 index 000000000..e9a8390c3 --- /dev/null +++ b/tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py @@ -0,0 +1,121 @@ +import functools +import unittest + +import numpy as np +import pandas as pd +from sklearn import metrics +from sklearn.linear_model import LogisticRegression + +import validmind as vm +from validmind.tests.model_validation.sklearn.OverfitDiagnosis import ( + OverfitDiagnosis, + _classification_metric_fn, +) + + +def _classification_datasets(input_id, labels, seed=0, n=160): + """Build train/test VMDatasets with a fitted classifier. + + Predictions and probabilities are computed by the model (not injected) so + ``OverfitDiagnosis`` detects a classification task via the probability + column. One feature is correlated with the class so feature bins carry a + subset of the classes. + """ + frames = [] + for offset in (0, 100): + rng = np.random.default_rng(seed + offset) + y = rng.choice(labels, size=n) + frames.append( + pd.DataFrame( + { + "f1": y + rng.normal(0, 0.4, n), + "f2": rng.normal(0, 1, n), + "target": y, + } + ) + ) + train_df, test_df = frames + + train_ds = vm.init_dataset( + input_id=f"{input_id}_train", + dataset=train_df, + target_column="target", + __log=False, + ) + test_ds = vm.init_dataset( + input_id=f"{input_id}_test", + dataset=test_df, + target_column="target", + __log=False, + ) + + model = LogisticRegression(max_iter=2000) + model.fit(train_df[["f1", "f2"]].to_numpy(), train_df["target"].to_numpy()) + vm_model = vm.init_model(input_id=f"{input_id}_model", model=model, __log=False) + + train_ds.assign_predictions(model=vm_model) + test_ds.assign_predictions(model=vm_model) + return train_ds, test_ds, vm_model + + +class TestOverfitClassificationMetricFn(unittest.TestCase): + """Unit tests for the averaging strategy selected from the global labels.""" + + def test_multiclass_labels_use_macro(self): + labels = np.array([0, 2, 4]) + fn = _classification_metric_fn("f1", labels) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.f1_score) + self.assertEqual(fn.keywords["average"], "macro") + self.assertEqual(fn.keywords["zero_division"], 0) + self.assertTrue(np.array_equal(fn.keywords["labels"], labels)) + + def test_non_standard_binary_uses_pos_label(self): + fn = _classification_metric_fn("precision", np.array([0, 4])) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.precision_score) + self.assertEqual(fn.keywords, {"pos_label": 4}) + + def test_conventional_binary_left_unchanged(self): + self.assertIs( + _classification_metric_fn("recall", np.array([0, 1])), metrics.recall_score + ) + + def test_non_prf_metrics_left_unchanged(self): + # auc/accuracy never take averaging kwargs and must not be wrapped. + self.assertIs( + _classification_metric_fn("auc", np.array([0, 2, 4])), + metrics.roc_auc_score, + ) + self.assertIs( + _classification_metric_fn("accuracy", np.array([0, 2, 4])), + metrics.accuracy_score, + ) + + +class TestOverfitDiagnosisMulticlass(unittest.TestCase): + """Regression tests for ZD-704 sibling exposure (explicit f1 selection).""" + + def test_multiclass_f1(self): + train_ds, test_ds, model = _classification_datasets( + "ovf_multi", [0, 2, 4], seed=1 + ) + result = OverfitDiagnosis( + model=model, datasets=[train_ds, test_ds], metric="f1" + ) + self.assertIn("Overfit Diagnosis", result[0]) + + def test_binary_f1_without_one(self): + train_ds, test_ds, model = _classification_datasets( + "ovf_binary04", [0, 4], seed=2 + ) + result = OverfitDiagnosis( + model=model, datasets=[train_ds, test_ds], metric="f1" + ) + self.assertIn("Overfit Diagnosis", result[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py new file mode 100644 index 000000000..233ab532a --- /dev/null +++ b/tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py @@ -0,0 +1,121 @@ +import functools +import unittest + +import numpy as np +import pandas as pd +from sklearn import metrics +from sklearn.linear_model import LogisticRegression + +import validmind as vm +from validmind.tests.model_validation.sklearn.RobustnessDiagnosis import ( + RobustnessDiagnosis, + _classification_metric_fn, +) + + +def _classification_datasets(input_id, labels, seed=0, n=160): + """Build train/test VMDatasets with a fitted classifier. + + ``RobustnessDiagnosis`` perturbs the numeric features and calls + ``model.predict`` on the noisy data, so a genuinely fitted model is needed + (predictions cannot be injected). Two numeric features are provided for the + noise to act on. + """ + frames = [] + for offset in (0, 100): + rng = np.random.default_rng(seed + offset) + y = rng.choice(labels, size=n) + frames.append( + pd.DataFrame( + { + "f1": y + rng.normal(0, 0.4, n), + "f2": rng.normal(0, 1, n), + "target": y, + } + ) + ) + train_df, test_df = frames + + train_ds = vm.init_dataset( + input_id=f"{input_id}_train", + dataset=train_df, + target_column="target", + __log=False, + ) + test_ds = vm.init_dataset( + input_id=f"{input_id}_test", + dataset=test_df, + target_column="target", + __log=False, + ) + + model = LogisticRegression(max_iter=2000) + model.fit(train_df[["f1", "f2"]].to_numpy(), train_df["target"].to_numpy()) + vm_model = vm.init_model(input_id=f"{input_id}_model", model=model, __log=False) + + train_ds.assign_predictions(model=vm_model) + test_ds.assign_predictions(model=vm_model) + return train_ds, test_ds, vm_model + + +class TestRobustnessClassificationMetricFn(unittest.TestCase): + """Unit tests for the averaging strategy selected from the global labels.""" + + def test_multiclass_labels_use_macro(self): + labels = np.array([0, 2, 4]) + fn = _classification_metric_fn("f1", labels) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.f1_score) + self.assertEqual(fn.keywords["average"], "macro") + self.assertEqual(fn.keywords["zero_division"], 0) + self.assertTrue(np.array_equal(fn.keywords["labels"], labels)) + + def test_non_standard_binary_uses_pos_label(self): + fn = _classification_metric_fn("recall", np.array([0, 4])) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.recall_score) + self.assertEqual(fn.keywords, {"pos_label": 4}) + + def test_conventional_binary_left_unchanged(self): + self.assertIs( + _classification_metric_fn("precision", np.array([0, 1])), + metrics.precision_score, + ) + + def test_non_prf_metrics_left_unchanged(self): + self.assertIs( + _classification_metric_fn("auc", np.array([0, 2, 4])), + metrics.roc_auc_score, + ) + self.assertIs( + _classification_metric_fn("accuracy", np.array([0, 2, 4])), + metrics.accuracy_score, + ) + + +class TestRobustnessDiagnosisMulticlass(unittest.TestCase): + """Regression tests for ZD-704 sibling exposure (explicit f1 selection).""" + + def test_multiclass_f1(self): + train_ds, test_ds, model = _classification_datasets( + "rbd_multi", [0, 2, 4], seed=1 + ) + result = RobustnessDiagnosis( + datasets=[train_ds, test_ds], model=model, metric="f1" + ) + self.assertIsInstance(result[0], pd.DataFrame) + + def test_binary_f1_without_one(self): + train_ds, test_ds, model = _classification_datasets( + "rbd_binary04", [0, 4], seed=2 + ) + result = RobustnessDiagnosis( + datasets=[train_ds, test_ds], model=model, metric="f1" + ) + self.assertIsInstance(result[0], pd.DataFrame) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py index ff29cc113..0be8db88a 100644 --- a/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py +++ b/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py @@ -1,10 +1,71 @@ +import functools import unittest +import numpy as np +import pandas as pd +from sklearn import metrics +from sklearn.linear_model import LogisticRegression + +import validmind as vm from validmind.tests.model_validation.sklearn.WeakspotsDiagnosis import ( + WeakspotsDiagnosis, + _averaged_default_metrics, _prepare_metrics_and_thresholds, ) +def _train_test_datasets(input_id, labels, seed=0, n=140): + """Build train/test VMDatasets and a fitted model for a given label set. + + Two numeric features are generated so ``WeakspotsDiagnosis`` has columns to + bin, with one feature correlated with the class so individual bins carry + only a subset of the classes (which is what makes per-slice averaging + crash). Predictions are injected verbatim so the true/predicted label sets + are controlled exactly and never accidentally collapse to {0, 1}. + """ + frames = [] + for offset in (0, 100): + rng = np.random.default_rng(seed + offset) + y = rng.choice(labels, size=n) + frames.append( + pd.DataFrame( + { + "f1": y + rng.normal(0, 0.4, n), + "f2": rng.normal(0, 1, n), + "target": y, + } + ) + ) + train_df, test_df = frames + + train_ds = vm.init_dataset( + input_id=f"{input_id}_train", + dataset=train_df, + target_column="target", + __log=False, + ) + test_ds = vm.init_dataset( + input_id=f"{input_id}_test", + dataset=test_df, + target_column="target", + __log=False, + ) + + model = LogisticRegression(max_iter=2000) + model.fit(train_df[["f1", "f2"]].to_numpy(), train_df["target"].to_numpy()) + vm_model = vm.init_model(input_id=f"{input_id}_model", model=model, __log=False) + + train_ds.assign_predictions( + model=vm_model, + prediction_values=model.predict(train_df[["f1", "f2"]].to_numpy()), + ) + test_ds.assign_predictions( + model=vm_model, + prediction_values=model.predict(test_df[["f1", "f2"]].to_numpy()), + ) + return train_ds, test_ds, vm_model + + class TestWeakspotsDiagnosisThresholds(unittest.TestCase): def test_partial_thresholds_use_defaults_for_plotting(self): _, plot_thresholds, pass_thresholds = _prepare_metrics_and_thresholds( @@ -27,5 +88,70 @@ def test_partial_thresholds_subset_for_pass_fail(self): self.assertEqual(set(pass_thresholds.keys()), {"Accuracy", "F1"}) +class TestWeakspotsDefaultMetricAveraging(unittest.TestCase): + """Unit tests for the averaging strategy selected from the global labels. + + Regression tests for ZD-704: sklearn's precision/recall/f1 default to + ``average="binary", pos_label=1`` which raises for multiclass labels and for + binary labels that don't contain 1 (e.g. {0, 4}). + """ + + def test_multiclass_labels_use_macro(self): + labels = np.array([0, 2, 4]) + bound = _averaged_default_metrics(labels) + + for name in ("precision", "recall", "f1"): + self.assertIsInstance(bound[name], functools.partial) + self.assertEqual(bound[name].keywords["average"], "macro") + self.assertEqual(bound[name].keywords["zero_division"], 0) + self.assertTrue(np.array_equal(bound[name].keywords["labels"], labels)) + # accuracy takes no averaging kwargs and must be left alone + self.assertIs(bound["accuracy"], metrics.accuracy_score) + + def test_non_standard_binary_uses_pos_label(self): + # Two classes, neither of which is 1 -- the customer's exact case. + bound = _averaged_default_metrics(np.array([0, 4])) + + for name in ("precision", "recall", "f1"): + self.assertIsInstance(bound[name], functools.partial) + self.assertEqual(bound[name].keywords, {"pos_label": 4}) + self.assertIs(bound["accuracy"], metrics.accuracy_score) + + def test_conventional_binary_left_unchanged(self): + bound = _averaged_default_metrics(np.array([0, 1])) + + self.assertIs(bound["precision"], metrics.precision_score) + self.assertIs(bound["recall"], metrics.recall_score) + self.assertIs(bound["f1"], metrics.f1_score) + self.assertIs(bound["accuracy"], metrics.accuracy_score) + + +class TestWeakspotsDiagnosisMulticlass(unittest.TestCase): + """End-to-end regression tests for ZD-704. + + On unpatched code these raise ``ValueError`` per feature-bin slice; the fix + decides the averaging once from the global label set so the test runs. + """ + + def test_multiclass_non_contiguous_labels(self): + train_ds, test_ds, model = _train_test_datasets("wsd_multi", [0, 2, 4], seed=1) + # Would raise "Target is multiclass but average='binary'" / "pos_label=1 + # is not a valid label" on a per-slice basis before the fix. + result = WeakspotsDiagnosis(datasets=[train_ds, test_ds], model=model) + self.assertGreater(len(result), 1) + + def test_binary_labels_without_one(self): + train_ds, test_ds, model = _train_test_datasets("wsd_binary04", [0, 4], seed=2) + # Would raise "pos_label=1 is not a valid label. It should be one of + # [0, 4]" before the fix -- the customer's reported error. + result = WeakspotsDiagnosis(datasets=[train_ds, test_ds], model=model) + self.assertGreater(len(result), 1) + + def test_conventional_binary_still_runs(self): + train_ds, test_ds, model = _train_test_datasets("wsd_binary01", [0, 1], seed=3) + result = WeakspotsDiagnosis(datasets=[train_ds, test_ds], model=model) + self.assertGreater(len(result), 1) + + if __name__ == "__main__": unittest.main() diff --git a/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py b/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py index ecff1006d..3ce6ea16b 100644 --- a/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py @@ -2,7 +2,8 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial -from typing import Dict, List, Tuple +import functools +from typing import Callable, Dict, List, Tuple import matplotlib.pyplot as plt import numpy as np @@ -64,6 +65,25 @@ } +def _classification_metric_fn(metric: str, labels: np.ndarray) -> Callable: + """Resolve the metric function, binding the averaging mode when needed. + + scikit-learn's precision/recall/f1 default to ``average="binary"`` with + ``pos_label=1``, which raises for multiclass labels and for binary labels + that do not include ``1`` (e.g. ``{0, 4}``). Following the MinimumF1Score fix + (PR #529), decide the averaging once from the global label set. Every other + metric (accuracy, auc, ...) is returned unchanged. + """ + fn = PERFORMANCE_METRICS[metric]["function"] + if metric not in ("f1", "precision", "recall"): + return fn + if len(labels) > 2: + return functools.partial(fn, average="macro", labels=labels, zero_division=0) + if 1 not in labels: + return functools.partial(fn, pos_label=labels.max()) + return fn + + def _prepare_results( results_train: dict, results_test: dict, metric: str ) -> pd.DataFrame: @@ -95,6 +115,7 @@ def _compute_metrics( pred_column: str, feature_column: str, metric: str, + metric_func: Callable, is_classification: bool, ) -> None: results["slice"].append(str(region)) @@ -106,7 +127,6 @@ def _compute_metrics( results[metric].append(0) return - metric_func = PERFORMANCE_METRICS[metric]["function"] y_true = df_region[target_column].values # AUC requires probability scores @@ -253,6 +273,24 @@ def OverfitDiagnosis( train_df[prob_column] = datasets[0].y_prob(model) test_df[prob_column] = datasets[1].y_prob(model) + # Decide the precision/recall/f1 averaging once from the labels sklearn will + # actually see -- the union of the target and prediction columns across both + # datasets -- so multiclass and non-{0, 1} binary models do not crash. + if is_classification: + labels = np.unique( + np.concatenate( + [ + train_df[datasets[0].target_column].values, + train_df[pred_column].values, + test_df[datasets[1].target_column].values, + test_df[pred_column].values, + ] + ) + ) + metric_func = _classification_metric_fn(metric, labels) + else: + metric_func = PERFORMANCE_METRICS[metric]["function"] + test_results = [] figures = [] results_headers = ["slice", "shape", "feature", metric] @@ -276,6 +314,7 @@ def OverfitDiagnosis( prob_column=prob_column, pred_column=pred_column, metric=metric, + metric_func=metric_func, is_classification=is_classification, ) df_test_region = test_df[ @@ -291,6 +330,7 @@ def OverfitDiagnosis( prob_column=prob_column, pred_column=pred_column, metric=metric, + metric_func=metric_func, is_classification=is_classification, ) diff --git a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py index b7639583d..f5871b539 100644 --- a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py @@ -2,9 +2,10 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +import functools from collections import defaultdict from operator import add -from typing import List, Tuple +from typing import Callable, List, Tuple import numpy as np import pandas as pd @@ -85,8 +86,31 @@ def _add_noise_std_dev( return noisy_values +def _classification_metric_fn(metric: str, labels: np.ndarray) -> Callable: + """Resolve the metric function, binding the averaging mode when needed. + + scikit-learn's precision/recall/f1 default to ``average="binary"`` with + ``pos_label=1``, which raises for multiclass labels and for binary labels + that do not include ``1`` (e.g. ``{0, 4}``). Following the MinimumF1Score fix + (PR #529), decide the averaging once from the global label set. Every other + metric (accuracy, auc, ...) is returned unchanged. + """ + fn = PERFORMANCE_METRICS[metric]["function"] + if metric not in ("f1", "precision", "recall"): + return fn + if len(labels) > 2: + return functools.partial(fn, average="macro", labels=labels, zero_division=0) + if 1 not in labels: + return functools.partial(fn, pos_label=labels.max()) + return fn + + def _compute_metric( - dataset: VMDataset, model: VMModel, X: pd.DataFrame, metric: str + dataset: VMDataset, + model: VMModel, + X: pd.DataFrame, + metric: str, + metric_func: Callable = None, ) -> float: if metric not in PERFORMANCE_METRICS: raise ValueError( @@ -100,7 +124,10 @@ def _compute_metric( y_proba = model.predict(X) return metrics.roc_auc_score(dataset.y, y_proba) - return PERFORMANCE_METRICS[metric]["function"](dataset.y, model.predict(X)) + if metric_func is None: + metric_func = PERFORMANCE_METRICS[metric]["function"] + + return metric_func(dataset.y, model.predict(X)) def _compute_gap(result: dict, metric: str) -> float: @@ -267,6 +294,19 @@ def RobustnessDiagnosis( else DEFAULT_REGRESSION_METRIC ) + # Decide the precision/recall/f1 averaging once from the unperturbed labels -- + # the union of y_true and the baseline predictions across datasets -- so + # multiclass and non-{0, 1} binary models do not crash. + metric_func = None + if metric in ("f1", "precision", "recall"): + labels = np.unique( + np.concatenate( + [np.asarray(dataset.y) for dataset in datasets] + + [np.asarray(model.predict(dataset.x_df())) for dataset in datasets] + ) + ) + metric_func = _classification_metric_fn(metric, labels) + results = [{} for _ in range(len(datasets))] # add baseline results (no perturbation) @@ -281,6 +321,7 @@ def RobustnessDiagnosis( model=model, X=dataset.x_df(), metric=metric, + metric_func=metric_func, ) ] result["Performance Decay"] = [0.0] @@ -307,6 +348,7 @@ def RobustnessDiagnosis( model=model, X=temp_df, metric=metric, + metric_func=metric_func, ) ) result["Performance Decay"].append(_compute_gap(result, metric)) @@ -325,9 +367,9 @@ def RobustnessDiagnosis( # rename perturbation size for baseline # Convert to object type first to avoid dtype incompatibility warning results_df["Perturbation Size"] = results_df["Perturbation Size"].astype(object) - results_df.loc[results_df["Perturbation Size"] == 0.0, "Perturbation Size"] = ( - "Baseline (0.0)" - ) + results_df.loc[ + results_df["Perturbation Size"] == 0.0, "Perturbation Size" + ] = "Baseline (0.0)" return ( results_df, diff --git a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py index c92fee871..6e287fb89 100644 --- a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py @@ -2,9 +2,11 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +import functools from typing import Callable, Dict, List, Optional, Tuple import matplotlib.pyplot as plt +import numpy as np import pandas as pd import plotly.graph_objects as go import seaborn as sns @@ -32,9 +34,37 @@ def _normalize_dict_keys(d: Dict) -> Dict: return {k.title(): v for k, v in d.items()} +def _averaged_default_metrics(labels: np.ndarray) -> Dict[str, Callable]: + """Bind the default precision/recall/f1 metrics to a suitable averaging mode. + + scikit-learn's precision/recall/f1 default to ``average="binary"`` with + ``pos_label=1``, which raises for multiclass labels and for binary labels + that do not include ``1`` (e.g. ``{0, 4}``). Following the MinimumF1Score fix + (PR #529), the averaging is decided once from the global label set so the + per-slice scores stay comparable: + + - multiclass (>2 labels): macro averaging over the global ``labels`` with + ``zero_division=0`` to suppress undefined-metric warnings for slices that + happen to lack some classes; + - binary but ``1`` is not a label: use the larger label as ``pos_label``; + - conventional binary ({0, 1}): leave scikit-learn's defaults untouched. + """ + averaged = dict(DEFAULT_METRICS) + if len(labels) > 2: + kwargs = {"average": "macro", "labels": labels, "zero_division": 0} + elif 1 not in labels: + kwargs = {"pos_label": labels.max()} + else: + return averaged + for name in ("precision", "recall", "f1"): + averaged[name] = functools.partial(averaged[name], **kwargs) + return averaged + + def _prepare_metrics_and_thresholds( metrics: Optional[Dict[str, Callable]], thresholds: Optional[Dict[str, float]], + labels: Optional[np.ndarray] = None, ) -> Tuple[Dict[str, Callable], Dict[str, float], Dict[str, float]]: """ Prepare metrics and threshold dicts for plotting and pass/fail checks. @@ -43,7 +73,15 @@ def _prepare_metrics_and_thresholds( Plotting uses default thresholds for any metric without an explicit value so charts always show a reference line; pass/fail uses only the user-provided thresholds when a custom dict is supplied. + + When the default metrics are used, ``labels`` (the global label set) selects + the averaging mode for precision/recall/f1 so multiclass and non-{0, 1} + binary models do not crash. Custom metric callables own their own kwargs and + are left untouched. """ + if metrics is None and labels is not None: + metrics = _averaged_default_metrics(labels) + normalized_metrics = _normalize_dict_keys(metrics or DEFAULT_METRICS) default_thresholds = _normalize_dict_keys(DEFAULT_THRESHOLDS) @@ -240,6 +278,8 @@ def WeakspotsDiagnosis( data types only. - Despite its usefulness in highlighting problematic regions, the test does not offer direct suggestions for model improvement. + - For multiclass models the default precision/recall/f1 metrics are macro-averaged over the global label set; for + binary models whose labels do not include ``1`` (e.g. ``{0, 4}``) the larger label is used as the positive class. """ feature_columns = features_columns or datasets[0].feature_columns numeric_and_categorical_columns = ( @@ -260,8 +300,32 @@ def WeakspotsDiagnosis( "Column(s) provided in features_columns do not exist in the dataset" ) + df_1 = datasets[0]._df[ + feature_columns + + [datasets[0].target_column, datasets[0].prediction_column(model)] + ] + df_2 = datasets[1]._df[ + feature_columns + + [datasets[1].target_column, datasets[1].prediction_column(model)] + ] + + # Decide the precision/recall/f1 averaging once from the labels sklearn will + # actually see -- the union of the target and prediction columns of both + # datasets. Deciding per slice would silently mix binary and macro semantics + # across bars of the same chart and still break on {0, 4}-style slices. + labels = np.unique( + np.concatenate( + [ + df_1[datasets[0].target_column].values, + df_1[datasets[0].prediction_column(model)].values, + df_2[datasets[1].target_column].values, + df_2[datasets[1].prediction_column(model)].values, + ] + ) + ) + metrics, plot_thresholds, pass_thresholds = _prepare_metrics_and_thresholds( - metrics, thresholds + metrics, thresholds, labels ) results_headers = ["Slice", "Number of Records", "Feature"] @@ -270,14 +334,6 @@ def WeakspotsDiagnosis( figures = [] passed = True - df_1 = datasets[0]._df[ - feature_columns - + [datasets[0].target_column, datasets[0].prediction_column(model)] - ] - df_2 = datasets[1]._df[ - feature_columns - + [datasets[1].target_column, datasets[1].prediction_column(model)] - ] results_1 = pd.DataFrame() results_2 = pd.DataFrame() for feature in feature_columns: