From d5ea308e54b29a02caa933a5e825bd6380dab1f0 Mon Sep 17 00:00:00 2001 From: Josh Stauffer <66793731+joshua-stauffer@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:28:27 +0200 Subject: [PATCH 1/5] Migrate statistical distribution expectations to the modular Expectations API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement and export four previously-stubbed statistical Expectations that raised NotImplementedError and were excluded from the public API: - ExpectColumnChisquareTestPValueToBeGreaterThan - ExpectColumnBootstrappedKsTestPValueToBeGreaterThan - ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan - ExpectColumnPairCramersPhiValueToBeLessThan Chi-square computes its statistic in _validate from column value counts and non-null count; the KS Expectations wire up their existing Pandas metric providers (registering the bootstrapped-KS provider and fixing its value_keys); Cramér's phi is a batch-level association measure over two columns computed from the full frame. Also fixes the parameterized-distribution KS metric to freeze the requested distribution and test against its CDF, which is required by current SciPy. Each Expectation is Pandas-only, matching the backends its underlying computation supports. Adds integration tests under data_sources_and_expectations and generated JSON schemas for each new Expectation. --- great_expectations/expectations/__init__.py | 4 + .../expectations/core/__init__.py | 12 + ...pped_ks_test_p_value_to_be_greater_than.py | 352 ++++++++++++- ...isquare_test_p_value_to_be_greater_than.py | 403 ++++++++++++++- ..._pair_cramers_phi_value_to_be_less_than.py | 436 ++++++++++++++--- ...tion_ks_test_p_value_to_be_greater_than.py | 341 ++++++++++++- ...otstrappedKsTestPValueToBeGreaterThan.json | 462 ++++++++++++++++++ ...umnChisquareTestPValueToBeGreaterThan.json | 453 +++++++++++++++++ ...ColumnPairCramersPhiValueToBeLessThan.json | 266 ++++++++++ ...stributionKsTestPValueToBeGreaterThan.json | 459 +++++++++++++++++ .../column_aggregate_metrics/__init__.py | 1 + .../column_bootstrapped_ks_test_p_value.py | 2 +- ...ameterized_distribution_ks_test_p_value.py | 7 +- tasks.py | 4 + ...pped_ks_test_p_value_to_be_greater_than.py | 71 +++ ...isquare_test_p_value_to_be_greater_than.py | 70 +++ ..._pair_cramers_phi_value_to_be_less_than.py | 82 ++++ ...tion_ks_test_p_value_to_be_greater_than.py | 67 +++ 18 files changed, 3377 insertions(+), 115 deletions(-) create mode 100644 great_expectations/expectations/core/schemas/ExpectColumnBootstrappedKsTestPValueToBeGreaterThan.json create mode 100644 great_expectations/expectations/core/schemas/ExpectColumnChisquareTestPValueToBeGreaterThan.json create mode 100644 great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json create mode 100644 great_expectations/expectations/core/schemas/ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan.json create mode 100644 tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py create mode 100644 tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py create mode 100644 tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py create mode 100644 tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py diff --git a/great_expectations/expectations/__init__.py b/great_expectations/expectations/__init__.py index 3f98bc88dcff..711d7b445429 100644 --- a/great_expectations/expectations/__init__.py +++ b/great_expectations/expectations/__init__.py @@ -1,6 +1,8 @@ from great_expectations.expectations.expectation import Expectation from .core import ( + ExpectColumnBootstrappedKsTestPValueToBeGreaterThan, + ExpectColumnChisquareTestPValueToBeGreaterThan, ExpectColumnDistinctValuesToBeInSet, ExpectColumnDistinctValuesToContainSet, ExpectColumnDistinctValuesToEqualSet, @@ -10,9 +12,11 @@ ExpectColumnMedianToBeBetween, ExpectColumnMinToBeBetween, ExpectColumnMostCommonValueToBeInSet, + ExpectColumnPairCramersPhiValueToBeLessThan, ExpectColumnPairValuesAToBeGreaterThanB, ExpectColumnPairValuesToBeEqual, ExpectColumnPairValuesToBeInSet, + ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan, ExpectColumnProportionOfNonNullValuesToBeBetween, ExpectColumnProportionOfUniqueValuesToBeBetween, ExpectColumnQuantileValuesToBeBetween, diff --git a/great_expectations/expectations/core/__init__.py b/great_expectations/expectations/core/__init__.py index 36e28920ee33..cb464572ab68 100644 --- a/great_expectations/expectations/core/__init__.py +++ b/great_expectations/expectations/core/__init__.py @@ -1,3 +1,9 @@ +from .expect_column_bootstrapped_ks_test_p_value_to_be_greater_than import ( + ExpectColumnBootstrappedKsTestPValueToBeGreaterThan, +) +from .expect_column_chisquare_test_p_value_to_be_greater_than import ( + ExpectColumnChisquareTestPValueToBeGreaterThan, +) from .expect_column_distinct_values_to_be_in_set import ( ExpectColumnDistinctValuesToBeInSet, ) @@ -17,11 +23,17 @@ from .expect_column_most_common_value_to_be_in_set import ( ExpectColumnMostCommonValueToBeInSet, ) +from .expect_column_pair_cramers_phi_value_to_be_less_than import ( + ExpectColumnPairCramersPhiValueToBeLessThan, +) from .expect_column_pair_values_a_to_be_greater_than_b import ( ExpectColumnPairValuesAToBeGreaterThanB, ) from .expect_column_pair_values_to_be_equal import ExpectColumnPairValuesToBeEqual from .expect_column_pair_values_to_be_in_set import ExpectColumnPairValuesToBeInSet +from .expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than import ( + ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan, +) from .expect_column_proportion_of_non_null_values_to_be_between import ( ExpectColumnProportionOfNonNullValuesToBeBetween, ) diff --git a/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py b/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py index 3bce0589a221..f696e02f38b3 100644 --- a/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py +++ b/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py @@ -1,33 +1,357 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union + +from great_expectations.compatibility import pydantic +from great_expectations.compatibility.typing_extensions import override +from great_expectations.core.suite_parameters import ( + SuiteParameterDict, # noqa: TC001 # FIXME CoP +) from great_expectations.expectations.expectation import ( - BatchExpectation, + ColumnAggregateExpectation, + _style_row_condition, + render_suite_parameter_string, +) +from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources +from great_expectations.expectations.model_field_descriptions import ( + COLUMN_DESCRIPTION, + FAILURE_SEVERITY_DESCRIPTION, +) +from great_expectations.render import ( + LegacyRendererType, + RenderedStringTemplateContent, +) +from great_expectations.render.renderer.renderer import renderer +from great_expectations.render.renderer_configuration import ( + RendererConfiguration, + RendererValueType, +) +from great_expectations.render.util import ( + parse_row_condition_string, + substitute_none_for_missing, +) + +if TYPE_CHECKING: + from great_expectations.core import ( + ExpectationValidationResult, + ) + from great_expectations.execution_engine import ExecutionEngine + from great_expectations.expectations.expectation_configuration import ( + ExpectationConfiguration, + ) + from great_expectations.render.renderer_configuration import AddParamArgs + +EXPECTATION_SHORT_DESCRIPTION = ( + "Expect the bootstrapped Kolmogorov-Smirnov test p-value statistic comparing the column to a " + "continuous partition object to be greater than a threshold." +) +PARTITION_OBJECT_DESCRIPTION = ( + "The expected continuous partition object, with finite ``bins`` and normalized ``weights`` and " + "no tail weights." ) +P_DESCRIPTION = ( + "The threshold below which a p-value counts as a failure. The Expectation succeeds when the " + "bootstrapped statistic is greater than this value. Defaults to 0.05." +) +BOOTSTRAP_SAMPLES_DESCRIPTION = ( + "The number of bootstrap rounds to perform. Defaults to 1000 when not provided." +) +BOOTSTRAP_SAMPLE_SIZE_DESCRIPTION = ( + "The number of elements to draw (with replacement) in each bootstrap round. Defaults to " + "twice the number of partition weights when not provided." +) +SUPPORTED_DATA_SOURCES = [ + SupportedDataSources.PANDAS.value, +] +DATA_QUALITY_ISSUES = [DataQualityIssues.NUMERIC.value] + + +class ExpectColumnBootstrappedKsTestPValueToBeGreaterThan(ColumnAggregateExpectation): + __doc__ = f"""{EXPECTATION_SHORT_DESCRIPTION} + + ExpectColumnBootstrappedKsTestPValueToBeGreaterThan is a \ + Column Aggregate Expectation. + + Column Aggregate Expectations are one of the most common types of Expectation. + They are evaluated for a single column, and produce an aggregate Metric, such as a mean, standard deviation, number of unique values, column type, etc. + If that Metric meets the conditions you set, the Expectation considers that data valid. + + This Expectation repeatedly draws bootstrap samples from the column and runs a one-sample \ + Kolmogorov-Smirnov test against the cumulative distribution function implied by the provided \ + continuous partition object. The reported statistic is the fraction of bootstrap rounds whose \ + KS p-value is at least ``p``; a high value indicates the data are consistent with the partition. + + Args: + column (str): \ + {COLUMN_DESCRIPTION} + partition_object (dict): \ + {PARTITION_OBJECT_DESCRIPTION} See [partition_object](https://docs.greatexpectations.io/docs/reference/expectations/distributional_expectations/#partition-objects). + p (float): \ + {P_DESCRIPTION} + bootstrap_samples (int or None): \ + {BOOTSTRAP_SAMPLES_DESCRIPTION} + bootstrap_sample_size (int or None): \ + {BOOTSTRAP_SAMPLE_SIZE_DESCRIPTION} + + Other Parameters: + result_format (str or None): \ + Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ + For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). + catch_exceptions (boolean or None): \ + If True, then catch exceptions and include them as part of the result object. \ + For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions). + meta (dict or None): \ + A JSON-serializable dictionary (nesting allowed) that will be included in the output without \ + modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta). + severity (str or None): \ + {FAILURE_SEVERITY_DESCRIPTION} \ + For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity). + + Returns: + An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result) + + Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta. + Notes: + * observed_value field in the result object is customized for this expectation to be the \ + bootstrapped p-value statistic (the fraction of bootstrap rounds whose KS p-value was at least ``p``). + * The Expectation succeeds when the observed statistic is greater than ``p``. + * Because the statistic is computed from random bootstrap samples, the observed_value may vary \ + slightly between runs. -# NOTE: This Expectation is incomplete and not ready for use. -# It should remain unexported until it meets the requirements set by our V1 API. -class ExpectColumnBootstrappedKsTestPValueToBeGreaterThan(BatchExpectation): - def __init__(self, *args, **kwargs): - raise NotImplementedError + Supported Data Sources: + [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) - library_metadata = { + Data Quality Issues: + {DATA_QUALITY_ISSUES[0]} + + Example Data: + test + 0 0.1 + 1 0.4 + 2 0.6 + 3 0.9 + 4 0.5 + + Code Examples: + Passing Case: + Input: + ExpectColumnBootstrappedKsTestPValueToBeGreaterThan( + column="test", + partition_object={{ + "bins": [0.0, 0.25, 0.5, 0.75, 1.0], + "weights": [0.25, 0.25, 0.25, 0.25], + }}, + p=0.05 + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 0.8 + }}, + "meta": {{}}, + "success": true + }} + + Failing Case: + Input: + ExpectColumnBootstrappedKsTestPValueToBeGreaterThan( + column="test", + partition_object={{ + "bins": [0.0, 0.25, 0.5, 0.75, 1.0], + "weights": [0.97, 0.01, 0.01, 0.01], + }}, + p=0.05 + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 0.0 + }}, + "meta": {{}}, + "success": false + }} + """ # noqa: E501 # FIXME CoP + + partition_object: Union[dict, SuiteParameterDict] = pydantic.Field( + description=PARTITION_OBJECT_DESCRIPTION + ) + p: Union[float, SuiteParameterDict] = pydantic.Field(default=0.05, description=P_DESCRIPTION) + bootstrap_samples: Union[int, SuiteParameterDict, None] = pydantic.Field( + default=None, description=BOOTSTRAP_SAMPLES_DESCRIPTION + ) + bootstrap_sample_size: Union[int, SuiteParameterDict, None] = pydantic.Field( + default=None, description=BOOTSTRAP_SAMPLE_SIZE_DESCRIPTION + ) + + library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { "maturity": "production", - "package": "great_expectations", "tags": [ "core expectation", "column aggregate expectation", - "needs migration to modular expectations api", + "distributional expectation", ], "contributors": ["@great_expectations"], "requirements": [], + "has_full_test_suite": True, + "manually_reviewed_code": True, } + _library_metadata = library_metadata - metric_dependencies = tuple() - success_keys = () + metric_dependencies = ("column.bootstrapped_ks_test_p_value",) + success_keys = ( + "partition_object", + "p", + "bootstrap_samples", + "bootstrap_sample_size", + ) args_keys = ( "column", - "distribution", - "p_value", - "params", + "partition_object", + "p", + "bootstrap_samples", + "bootstrap_sample_size", ) + + class Config: + title = "Expect column bootstrapped KS test p-value to be greater than" + + @staticmethod + def schema_extra( + schema: Dict[str, Any], + model: Type[ExpectColumnBootstrappedKsTestPValueToBeGreaterThan], + ) -> None: + ColumnAggregateExpectation.Config.schema_extra(schema, model) + schema["properties"]["metadata"]["properties"].update( + { + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": DATA_QUALITY_ISSUES, + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": model._library_metadata, + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": EXPECTATION_SHORT_DESCRIPTION, + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": SUPPORTED_DATA_SOURCES, + }, + } + ) + + @classmethod + @override + def _prescriptive_template( + cls, + renderer_configuration: RendererConfiguration, + ) -> RendererConfiguration: + add_param_args: AddParamArgs = ( + ("column", RendererValueType.STRING), + ("p", RendererValueType.NUMBER), + ) + for name, param_type in add_param_args: + renderer_configuration.add_param(name=name, param_type=param_type) + + template_str = ( + "bootstrapped Kolmogorov-Smirnov test p-value statistic must be greater than $p." + ) + if renderer_configuration.include_column_name: + template_str = f"$column {template_str}" + + renderer_configuration.template_str = template_str + return renderer_configuration + + @classmethod + @override + @renderer(renderer_type=LegacyRendererType.PRESCRIPTIVE) + @render_suite_parameter_string + def _prescriptive_renderer( # type: ignore[override] # TODO: Fix this type ignore + cls, + configuration: ExpectationConfiguration, + result: Optional[ExpectationValidationResult] = None, + runtime_configuration: Optional[dict] = None, + **kwargs, + ): + runtime_configuration = runtime_configuration or {} + include_column_name = runtime_configuration.get("include_column_name") is not False + styling = runtime_configuration.get("styling") + params = substitute_none_for_missing( + configuration.kwargs, + [ + "column", + "p", + "row_condition", + "condition_parser", + ], + ) + + template_str = ( + "bootstrapped Kolmogorov-Smirnov test p-value statistic must be greater than $p." + ) + if include_column_name: + template_str = f"$column {template_str}" + + if params["row_condition"] is not None: + conditional_template_str = parse_row_condition_string(params["row_condition"]) + template_str, styling = _style_row_condition( + conditional_template_str, + template_str, + params, + styling, + ) + + return [ + RenderedStringTemplateContent( + content_block_type="string_template", + string_template={ + "template": template_str, + "params": params, + "styling": styling, + }, + ) + ] + + @override + def _validate( + self, + metrics: Dict, + runtime_configuration: Optional[dict] = None, + execution_engine: Optional[ExecutionEngine] = None, + ): + configuration = self.configuration + p = configuration.kwargs.get("p", self._get_default_value("p")) + + metric_result = metrics["column.bootstrapped_ks_test_p_value"] + observed_value = float(metric_result["observed_value"]) + details = metric_result.get("details", {}) + + return { + "success": bool(observed_value > p), + "result": { + "observed_value": observed_value, + "details": { + "bootstrap_samples": details.get("bootstrap_samples"), + "bootstrap_sample_size": details.get("bootstrap_sample_size"), + }, + }, + } diff --git a/great_expectations/expectations/core/expect_column_chisquare_test_p_value_to_be_greater_than.py b/great_expectations/expectations/core/expect_column_chisquare_test_p_value_to_be_greater_than.py index 6153149edd1c..1acf931cb1ff 100644 --- a/great_expectations/expectations/core/expect_column_chisquare_test_p_value_to_be_greater_than.py +++ b/great_expectations/expectations/core/expect_column_chisquare_test_p_value_to_be_greater_than.py @@ -1,32 +1,415 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union + +import pandas as pd +from scipy import stats + +from great_expectations.compatibility import pydantic +from great_expectations.compatibility.typing_extensions import override +from great_expectations.core.suite_parameters import ( + SuiteParameterDict, # noqa: TC001 # FIXME CoP +) +from great_expectations.execution_engine.util import ( + is_valid_categorical_partition_object, +) from great_expectations.expectations.expectation import ( - BatchExpectation, + ColumnAggregateExpectation, + _style_row_condition, + render_suite_parameter_string, +) +from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources +from great_expectations.expectations.model_field_descriptions import ( + COLUMN_DESCRIPTION, + FAILURE_SEVERITY_DESCRIPTION, +) +from great_expectations.render import ( + LegacyRendererType, + RenderedStringTemplateContent, +) +from great_expectations.render.renderer.renderer import renderer +from great_expectations.render.renderer_configuration import ( + RendererConfiguration, + RendererValueType, +) +from great_expectations.render.util import ( + parse_row_condition_string, + substitute_none_for_missing, +) +from great_expectations.validator.metric_configuration import MetricConfiguration + +if TYPE_CHECKING: + from great_expectations.core import ( + ExpectationValidationResult, + ) + from great_expectations.execution_engine import ExecutionEngine + from great_expectations.expectations.expectation_configuration import ( + ExpectationConfiguration, + ) + from great_expectations.render.renderer_configuration import AddParamArgs + from great_expectations.validator.validator import ValidationDependencies + +EXPECTATION_SHORT_DESCRIPTION = ( + "Expect the p-value of a Chi-square goodness-of-fit test comparing the observed categorical " + "frequencies of the column to an expected partition object to be greater than a threshold." +) +PARTITION_OBJECT_DESCRIPTION = ( + "The expected categorical partition object, with ``values`` and normalized ``weights``." ) +P_DESCRIPTION = ( + "The threshold p-value. The Expectation succeeds when the observed Chi-square p-value is " + "greater than this value. Defaults to 0.05." +) +TAIL_WEIGHT_HOLDOUT_DESCRIPTION = ( + "The amount of weight to split uniformly among values observed in the data but absent from " + "the partition object. Provides a mechanism to make the test less strict. Defaults to 0." +) +SUPPORTED_DATA_SOURCES = [ + SupportedDataSources.PANDAS.value, +] +DATA_QUALITY_ISSUES = [DataQualityIssues.NUMERIC.value] + + +class ExpectColumnChisquareTestPValueToBeGreaterThan(ColumnAggregateExpectation): + __doc__ = f"""{EXPECTATION_SHORT_DESCRIPTION} + + ExpectColumnChisquareTestPValueToBeGreaterThan is a \ + Column Aggregate Expectation. + + Column Aggregate Expectations are one of the most common types of Expectation. + They are evaluated for a single column, and produce an aggregate Metric, such as a mean, standard deviation, number of unique values, column type, etc. + If that Metric meets the conditions you set, the Expectation considers that data valid. + + The Chi-square goodness-of-fit test compares the observed value counts of the column with the \ + expected counts implied by the partition object. A large p-value indicates that the observed \ + distribution is consistent with the expected one. + + Args: + column (str): \ + {COLUMN_DESCRIPTION} + partition_object (dict): \ + {PARTITION_OBJECT_DESCRIPTION} See [partition_object](https://docs.greatexpectations.io/docs/reference/expectations/distributional_expectations/#partition-objects). + p (float): \ + {P_DESCRIPTION} + tail_weight_holdout (float): \ + {TAIL_WEIGHT_HOLDOUT_DESCRIPTION} + + Other Parameters: + result_format (str or None): \ + Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ + For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). + catch_exceptions (boolean or None): \ + If True, then catch exceptions and include them as part of the result object. \ + For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions). + meta (dict or None): \ + A JSON-serializable dictionary (nesting allowed) that will be included in the output without \ + modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta). + severity (str or None): \ + {FAILURE_SEVERITY_DESCRIPTION} \ + For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity). + + Returns: + An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result) + + Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta. + + Notes: + * observed_value field in the result object is customized for this expectation to be the \ + p-value of the Chi-square test. + * details.observed_partition and details.expected_partition are customized for this \ + expectation to be dicts representing the observed and expected partitions. + * The Expectation succeeds when the observed p-value is greater than ``p``. + + Supported Data Sources: + [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) + + Data Quality Issues: + {DATA_QUALITY_ISSUES[0]} + Example Data: + test + 0 "A" + 1 "A" + 2 "B" + 3 "B" + 4 "C" -# NOTE: This Expectation is incomplete and not ready for use. -# It should remain unexported until it meets the requirements set by our V1 API. -class ExpectColumnChiSquareTestPValueToBeGreaterThan(BatchExpectation): - def __init__(self, *args, **kwargs): - raise NotImplementedError + Code Examples: + Passing Case: + Input: + ExpectColumnChisquareTestPValueToBeGreaterThan( + column="test", + partition_object={{"values": ["A", "B", "C"], "weights": [0.4, 0.4, 0.2]}}, + p=0.05 + ) - library_metadata = { + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 1.0 + }}, + "meta": {{}}, + "success": true + }} + + Failing Case: + Input: + ExpectColumnChisquareTestPValueToBeGreaterThan( + column="test", + partition_object={{"values": ["A", "B", "C"], "weights": [0.05, 0.05, 0.9]}}, + p=0.05 + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 0.0001 + }}, + "meta": {{}}, + "success": false + }} + """ # noqa: E501 # FIXME CoP + + partition_object: Union[dict, SuiteParameterDict] = pydantic.Field( + description=PARTITION_OBJECT_DESCRIPTION + ) + p: Union[float, SuiteParameterDict] = pydantic.Field(default=0.05, description=P_DESCRIPTION) + tail_weight_holdout: Union[float, SuiteParameterDict] = pydantic.Field( + default=0, ge=0, le=1, description=TAIL_WEIGHT_HOLDOUT_DESCRIPTION + ) + + library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", "column aggregate expectation", - "needs migration to modular expectations api", + "distributional expectation", ], "contributors": ["@great_expectations"], "requirements": [], + "has_full_test_suite": True, + "manually_reviewed_code": True, } + _library_metadata = library_metadata - metric_dependencies = tuple() - success_keys = () + success_keys = ( + "partition_object", + "p", + "tail_weight_holdout", + ) args_keys = ( "column", "partition_object", "p", "tail_weight_holdout", ) + + class Config: + title = "Expect column Chi-square test p-value to be greater than" + + @staticmethod + def schema_extra( + schema: Dict[str, Any], + model: Type[ExpectColumnChisquareTestPValueToBeGreaterThan], + ) -> None: + ColumnAggregateExpectation.Config.schema_extra(schema, model) + schema["properties"]["metadata"]["properties"].update( + { + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": DATA_QUALITY_ISSUES, + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": model._library_metadata, + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": EXPECTATION_SHORT_DESCRIPTION, + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": SUPPORTED_DATA_SOURCES, + }, + } + ) + + @classmethod + @override + def _prescriptive_template( + cls, + renderer_configuration: RendererConfiguration, + ) -> RendererConfiguration: + add_param_args: AddParamArgs = ( + ("column", RendererValueType.STRING), + ("p", RendererValueType.NUMBER), + ) + for name, param_type in add_param_args: + renderer_configuration.add_param(name=name, param_type=param_type) + + template_str = "Chi-square test p-value must be greater than $p." + if renderer_configuration.include_column_name: + template_str = f"$column {template_str}" + + renderer_configuration.template_str = template_str + return renderer_configuration + + @classmethod + @override + @renderer(renderer_type=LegacyRendererType.PRESCRIPTIVE) + @render_suite_parameter_string + def _prescriptive_renderer( # type: ignore[override] # TODO: Fix this type ignore + cls, + configuration: ExpectationConfiguration, + result: Optional[ExpectationValidationResult] = None, + runtime_configuration: Optional[dict] = None, + **kwargs, + ): + runtime_configuration = runtime_configuration or {} + include_column_name = runtime_configuration.get("include_column_name") is not False + styling = runtime_configuration.get("styling") + params = substitute_none_for_missing( + configuration.kwargs, + [ + "column", + "p", + "row_condition", + "condition_parser", + ], + ) + + template_str = "Chi-square test p-value must be greater than $p." + if include_column_name: + template_str = f"$column {template_str}" + + if params["row_condition"] is not None: + conditional_template_str = parse_row_condition_string(params["row_condition"]) + template_str, styling = _style_row_condition( + conditional_template_str, + template_str, + params, + styling, + ) + + return [ + RenderedStringTemplateContent( + content_block_type="string_template", + string_template={ + "template": template_str, + "params": params, + "styling": styling, + }, + ) + ] + + @override + def get_validation_dependencies( + self, + execution_engine: Optional[ExecutionEngine] = None, + runtime_configuration: Optional[dict] = None, + ) -> ValidationDependencies: + validation_dependencies: ValidationDependencies = super().get_validation_dependencies( + execution_engine, runtime_configuration + ) + domain_kwargs = self.configuration.get_domain_kwargs() + validation_dependencies.set_metric_configuration( + metric_name="column.value_counts", + metric_configuration=MetricConfiguration( + metric_name="column.value_counts", + metric_domain_kwargs=domain_kwargs, + metric_value_kwargs={"sort": "value"}, + ), + ) + validation_dependencies.set_metric_configuration( + metric_name="column_values.nonnull.count", + metric_configuration=MetricConfiguration( + metric_name="column_values.nonnull.count", + metric_domain_kwargs=domain_kwargs, + metric_value_kwargs=None, + ), + ) + return validation_dependencies + + @override + def _validate( + self, + metrics: Dict, + runtime_configuration: Optional[dict] = None, + execution_engine: Optional[ExecutionEngine] = None, + ): + configuration = self.configuration + partition_object = configuration.kwargs.get( + "partition_object", self._get_default_value("partition_object") + ) + p = configuration.kwargs.get("p", self._get_default_value("p")) + tail_weight_holdout = configuration.kwargs.get( + "tail_weight_holdout", self._get_default_value("tail_weight_holdout") + ) + + if not is_valid_categorical_partition_object(partition_object): + raise ValueError("Invalid categorical partition object.") # noqa: TRY003 # FIXME CoP + + element_count = metrics["column_values.nonnull.count"] + observed_frequencies = metrics["column.value_counts"] + # Convert to Series object to allow joining on index values + expected_column = ( + pd.Series( + partition_object["weights"], + index=partition_object["values"], + name="expected", + ) + * element_count + ) + # Join along the indices to allow proper comparison of both types of possible missing values + test_df = pd.concat([expected_column, observed_frequencies], axis=1) + + na_counts = test_df.isnull().sum() + + # Handle NaN: if we expected something that's not there, it's just not there. + test_df["count"] = test_df["count"].fillna(0) + # Handle NaN: if something's there that was not expected, substitute the relevant value + # for tail_weight_holdout + if na_counts["expected"] > 0: + # Scale existing expected values + test_df["expected"] *= 1 - tail_weight_holdout + # Fill NAs with holdout. + test_df["expected"] = test_df["expected"].fillna( + element_count * (tail_weight_holdout / na_counts["expected"]) + ) + + test_result = float(stats.chisquare(test_df["count"], test_df["expected"])[1]) + + # Normalize the outputs so they can be used as partitions into other expectations + expected_weights = (test_df["expected"] / test_df["expected"].sum()).tolist() + observed_weights = (test_df["count"] / test_df["count"].sum()).tolist() + + return { + "success": bool(test_result > p), + "result": { + "observed_value": test_result, + "details": { + "observed_partition": { + "values": test_df.index.tolist(), + "weights": observed_weights, + }, + "expected_partition": { + "values": test_df.index.tolist(), + "weights": expected_weights, + }, + }, + }, + } diff --git a/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py b/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py index bb004030a35f..7de0df155a15 100644 --- a/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py +++ b/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py @@ -1,74 +1,332 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +import itertools +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +import numpy as np +import pandas as pd +from scipy import stats + +from great_expectations.compatibility import pydantic +from great_expectations.compatibility.typing_extensions import override +from great_expectations.core.suite_parameters import ( + SuiteParameterDict, # noqa: TC001 # FIXME CoP +) from great_expectations.expectations.expectation import ( BatchExpectation, render_suite_parameter_string, ) +from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources +from great_expectations.expectations.model_field_descriptions import ( + FAILURE_SEVERITY_DESCRIPTION, +) from great_expectations.render import ( - LegacyDiagnosticRendererType, LegacyRendererType, RenderedStringTemplateContent, - RenderedTableContent, ) from great_expectations.render.renderer.renderer import renderer from great_expectations.render.renderer_configuration import ( RendererConfiguration, RendererValueType, ) -from great_expectations.render.util import num_to_str, substitute_none_for_missing +from great_expectations.render.util import substitute_none_for_missing +from great_expectations.validator.metric_configuration import MetricConfiguration if TYPE_CHECKING: from great_expectations.core import ( ExpectationValidationResult, ) + from great_expectations.execution_engine import ExecutionEngine from great_expectations.expectations.expectation_configuration import ( ExpectationConfiguration, ) from great_expectations.render.renderer_configuration import AddParamArgs + from great_expectations.validator.validator import ValidationDependencies + +EXPECTATION_SHORT_DESCRIPTION = ( + "Expect the Cramér's phi (V) measure of association between two columns to be less than or " + "equal to a threshold." +) +COLUMN_A_DESCRIPTION = "The first column name." +COLUMN_B_DESCRIPTION = "The second column name." +THRESHOLD_DESCRIPTION = ( + "The maximum Cramér's phi value for which to return success=True. Cramér's phi ranges from 0 " + "(no association / independent) to 1 (perfect association). Defaults to 0.1." +) +BINS_A_DESCRIPTION = ( + "Explicit bin edges (numeric column) or groups of values (categorical column) used to " + "discretize column_A before building the contingency table." +) +BINS_B_DESCRIPTION = "As bins_A, but for column_B." +N_BINS_A_DESCRIPTION = ( + "The number of bins to use for column_A when bins_A is not provided. Defaults to 10." +) +N_BINS_B_DESCRIPTION = "As n_bins_A, but for column_B." +SUPPORTED_DATA_SOURCES = [ + SupportedDataSources.PANDAS.value, +] +DATA_QUALITY_ISSUES = [DataQualityIssues.NUMERIC.value] + + +def _get_binned_values( # noqa: C901, PLR0912 # FIXME CoP + series: pd.Series, bins: Optional[list], n_bins: Optional[int] +): + """Get binned values of a series, binning numeric data into intervals and collapsing rare + categorical values, so that a contingency table can be built. + """ + if n_bins is None: + n_bins = 10 + + if series.dtype in ["int", "float", "int64", "float64"]: + if bins is not None: + sorted_bins = sorted(np.unique(bins)) + if np.min(series) < sorted_bins[0]: + sorted_bins = [np.min(series), *sorted_bins] + if np.max(series) > sorted_bins[-1]: + sorted_bins = [*sorted_bins, np.max(series)] + edges = np.array(sorted_bins, dtype=float) + else: + edges = np.array( + np.histogram_bin_edges(series[series.notnull()], bins=n_bins), dtype=float + ) + + # Make sure max of series is included in rightmost bin + edges[-1] = np.nextafter(edges[-1], edges[-1] + 1) + + # Create labels for the returned series + precision = int(np.log10(min(edges[1:] - edges[:-1]))) + 2 + labels = [ + f"[{round(lower, precision)}, {round(upper, precision)})" + for lower, upper in itertools.pairwise(edges) + ] + if series.isnull().any(): + # Missing get digitized into bin = n_bins + 1 + labels += ["(missing)"] + + return pd.Categorical.from_codes( + codes=np.digitize(series, bins=edges) - 1, + categories=pd.Index(labels), + ordered=True, + ) + + else: + if bins is None: + value_counts = series.value_counts(sort=True) + if len(value_counts) < n_bins + 1: + return series.fillna("(missing)") + else: + other_values = sorted(value_counts.index[n_bins:]) + replace = dict.fromkeys(other_values, "(other)") + else: + replace = {} + for x in bins: + replace.update({value: ", ".join(x) for value in x}) + return series.replace(to_replace=replace).fillna("(missing)").astype("category") + + +def _get_crosstab( # noqa: PLR0913 # FIXME CoP + series_A: pd.Series, + series_B: pd.Series, + bins_A: Optional[list], + bins_B: Optional[list], + n_bins_A: Optional[int], + n_bins_B: Optional[int], +) -> pd.DataFrame: + """Get the contingency table (crosstab) of two series, binning values if necessary.""" + binned_A = _get_binned_values(series_A, bins_A, n_bins_A) + binned_B = _get_binned_values(series_B, bins_B, n_bins_B) + return pd.crosstab(binned_A, columns=binned_B) -# NOTE: This Expectation is incomplete and not ready for use. -# It should remain unexported until it meets the requirements set by our V1 API. class ExpectColumnPairCramersPhiValueToBeLessThan(BatchExpectation): - def __init__(self, *args, **kwargs): - raise NotImplementedError + __doc__ = f"""{EXPECTATION_SHORT_DESCRIPTION} + + ExpectColumnPairCramersPhiValueToBeLessThan is a \ + Batch Expectation. + + BatchExpectations are one of the most common types of Expectation. + They are evaluated for an entire Batch, and answer a semantic question about the Batch itself. + + Cramér's phi (also written Cramér's V) measures the strength of association between two \ + categorical variables, derived from the Chi-square statistic of their contingency table. \ + A value near 0 indicates the columns are (close to) independent; larger values indicate \ + stronger association. Numeric columns are discretized into bins before the table is built. + + Args: + column_A (str): \ + {COLUMN_A_DESCRIPTION} + column_B (str): \ + {COLUMN_B_DESCRIPTION} + threshold (float): \ + {THRESHOLD_DESCRIPTION} + bins_A (list or None): \ + {BINS_A_DESCRIPTION} + bins_B (list or None): \ + {BINS_B_DESCRIPTION} + n_bins_A (int or None): \ + {N_BINS_A_DESCRIPTION} + n_bins_B (int or None): \ + {N_BINS_B_DESCRIPTION} + + Other Parameters: + result_format (str or None): \ + Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ + For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). + catch_exceptions (boolean or None): \ + If True, then catch exceptions and include them as part of the result object. \ + For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions). + meta (dict or None): \ + A JSON-serializable dictionary (nesting allowed) that will be included in the output without \ + modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta). + severity (str or None): \ + {FAILURE_SEVERITY_DESCRIPTION} \ + For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity). + + Returns: + An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result) + + Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta. + + Notes: + * observed_value field in the result object is customized for this expectation to be the \ + Cramér's phi (V) value. + * details.crosstab is customized for this expectation to be a serializable representation \ + of the contingency table between column_A and column_B. + * The Expectation succeeds when the observed Cramér's phi is less than or equal to threshold. + + Supported Data Sources: + [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) + + Data Quality Issues: + {DATA_QUALITY_ISSUES[0]} - library_metadata = { + Example Data: + test test2 + 0 "A" "X" + 1 "A" "Y" + 2 "B" "X" + 3 "B" "Y" + + Code Examples: + Passing Case: + Input: + ExpectColumnPairCramersPhiValueToBeLessThan( + column_A="test", + column_B="test2", + threshold=0.1 + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 0.0 + }}, + "meta": {{}}, + "success": true + }} + + Failing Case: + Input: + ExpectColumnPairCramersPhiValueToBeLessThan( + column_A="test", + column_B="test2", + threshold=0.1 + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 1.0 + }}, + "meta": {{}}, + "success": false + }} + """ # noqa: E501 # FIXME CoP + + column_A: str = pydantic.Field(min_length=1, description=COLUMN_A_DESCRIPTION) + column_B: str = pydantic.Field(min_length=1, description=COLUMN_B_DESCRIPTION) + threshold: Union[float, SuiteParameterDict] = pydantic.Field( + default=0.1, description=THRESHOLD_DESCRIPTION + ) + bins_A: Optional[List[Any]] = pydantic.Field(default=None, description=BINS_A_DESCRIPTION) + bins_B: Optional[List[Any]] = pydantic.Field(default=None, description=BINS_B_DESCRIPTION) + n_bins_A: Optional[int] = pydantic.Field(default=None, description=N_BINS_A_DESCRIPTION) + n_bins_B: Optional[int] = pydantic.Field(default=None, description=N_BINS_B_DESCRIPTION) + + library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", "multi-column expectation", - "needs migration to modular expectations api", + "distributional expectation", ], "contributors": ["@great_expectations"], "requirements": [], + "has_full_test_suite": True, + "manually_reviewed_code": True, } + _library_metadata = library_metadata - metric_dependencies = tuple() success_keys = ( "column_A", "column_B", "threshold", + "bins_A", + "bins_B", + "n_bins_A", + "n_bins_B", ) - # default_kwarg_values = { - # "column_A": None, - # "column_B": None, - # "bins_A": None, - # "bins_B": None, - # "n_bins_A": None, - # "n_bins_B": None, - # "threshold": 0.1, - # "result_format": "BASIC", - # "catch_exceptions": False, - # } args_keys = ( "column_A", "column_B", ) + class Config: + title = "Expect column pair Cramér's phi value to be less than" + + @staticmethod + def schema_extra( + schema: Dict[str, Any], + model: Type[ExpectColumnPairCramersPhiValueToBeLessThan], + ) -> None: + BatchExpectation.Config.schema_extra(schema, model) + schema["properties"]["metadata"]["properties"].update( + { + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": DATA_QUALITY_ISSUES, + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": model._library_metadata, + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": EXPECTATION_SHORT_DESCRIPTION, + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": SUPPORTED_DATA_SOURCES, + }, + } + ) + @classmethod + @override def _prescriptive_template( cls, renderer_configuration: RendererConfiguration, @@ -81,89 +339,113 @@ def _prescriptive_template( renderer_configuration.add_param(name=name, param_type=param_type) params = renderer_configuration.params - if not params.column_A or not params.column_B: - renderer_configuration.template_str = " unrecognized kwargs for expect_column_pair_cramers_phi_value_to_be_less_than: missing column." # noqa: E501 # FIXME CoP + renderer_configuration.template_str = ( + "Cramér's phi association requires two columns: missing column." + ) else: renderer_configuration.template_str = ( "Values in $column_A and $column_B must be independent." ) - return renderer_configuration @classmethod + @override @renderer(renderer_type=LegacyRendererType.PRESCRIPTIVE) @render_suite_parameter_string - def _prescriptive_renderer( + def _prescriptive_renderer( # type: ignore[override] # TODO: Fix this type ignore cls, - configuration: Optional[ExpectationConfiguration] = None, + configuration: ExpectationConfiguration, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, ): runtime_configuration = runtime_configuration or {} - _ = runtime_configuration.get("include_column_name") is not False styling = runtime_configuration.get("styling") params = substitute_none_for_missing(configuration.kwargs, ["column_A", "column_B"]) if (params["column_A"] is None) or (params["column_B"] is None): - template_str = " unrecognized kwargs for expect_column_pair_cramers_phi_value_to_be_less_than: missing column." # noqa: E501 # FIXME CoP + template_str = "Cramér's phi association requires two columns: missing column." else: template_str = "Values in $column_A and $column_B must be independent." - rendered_string_template_content = RenderedStringTemplateContent( - **{ - "content_block_type": "string_template", - "string_template": { + return [ + RenderedStringTemplateContent( + content_block_type="string_template", + string_template={ "template": template_str, "params": params, "styling": styling, }, - } - ) + ) + ] - return [rendered_string_template_content] + @override + def get_validation_dependencies( + self, + execution_engine: Optional[ExecutionEngine] = None, + runtime_configuration: Optional[dict] = None, + ) -> ValidationDependencies: + validation_dependencies: ValidationDependencies = super().get_validation_dependencies( + execution_engine, runtime_configuration + ) + domain_kwargs = self.configuration.get_domain_kwargs() + validation_dependencies.set_metric_configuration( + metric_name="table.head", + metric_configuration=MetricConfiguration( + metric_name="table.head", + metric_domain_kwargs=domain_kwargs, + metric_value_kwargs={"n_rows": None, "fetch_all": True}, + ), + ) + return validation_dependencies - @classmethod - @renderer(renderer_type=LegacyDiagnosticRendererType.OBSERVED_VALUE) - def _diagnostic_observed_value_renderer( - cls, - configuration: Optional[ExpectationConfiguration] = None, - result: Optional[ExpectationValidationResult] = None, + @override + def _validate( + self, + metrics: Dict, runtime_configuration: Optional[dict] = None, - **kwargs, + execution_engine: Optional[ExecutionEngine] = None, ): - observed_value = result.result.get("observed_value") - column_A = result.expectation_config.kwargs["column_A"] - column_B = result.expectation_config.kwargs["column_B"] - crosstab = result.result.get("details", {}).get("crosstab") - - if observed_value is not None: - observed_value = num_to_str(observed_value, precision=3, use_locale=True) - if crosstab is not None: - table = [[""] + list(crosstab.columns)] - for col in range(len(crosstab)): - table.append([crosstab.index[col]] + list(crosstab.iloc[col, :])) - - return RenderedTableContent( - **{ - "content_block_type": "table", - "header": f"Observed cramers phi of {observed_value}. \n" - f"Crosstab between {column_A} (rows) and {column_B} (columns):", - "table": table, - "styling": { - "body": { - "classes": [ - "table", - "table-sm", - "table-unbordered", - "col-4", - "mt-2", - ], - } - }, - } - ) - else: - return observed_value + configuration = self.configuration + column_A = configuration.kwargs.get("column_A", self._get_default_value("column_A")) + column_B = configuration.kwargs.get("column_B", self._get_default_value("column_B")) + threshold = configuration.kwargs.get("threshold", self._get_default_value("threshold")) + bins_A = configuration.kwargs.get("bins_A", self._get_default_value("bins_A")) + bins_B = configuration.kwargs.get("bins_B", self._get_default_value("bins_B")) + n_bins_A = configuration.kwargs.get("n_bins_A", self._get_default_value("n_bins_A")) + n_bins_B = configuration.kwargs.get("n_bins_B", self._get_default_value("n_bins_B")) + + df: pd.DataFrame = metrics["table.head"] + for column in (column_A, column_B): + if column not in df.columns: + raise ValueError(f"Column '{column}' not found in batch.") # noqa: TRY003 # FIXME CoP + + crosstab = _get_crosstab(df[column_A], df[column_B], bins_A, bins_B, n_bins_A, n_bins_B) + + counts = crosstab.to_numpy() + n = float(counts.sum()) + min_dimension = min(crosstab.shape) + + if n == 0 or min_dimension < 2: # noqa: PLR2004 # FIXME CoP + # Association is undefined / cannot exceed threshold when one variable is constant. + cramers_phi = 0.0 else: - return "--" + chi2_statistic = stats.chi2_contingency(counts)[0] + # See e.g. https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V + cramers_phi = float(max(min(np.sqrt(chi2_statistic / n / (min_dimension - 1)), 1), 0)) + + return { + "success": bool(cramers_phi <= threshold), + "result": { + "observed_value": cramers_phi, + "details": { + "crosstab": { + "row_variable": column_A, + "column_variable": column_B, + "rows": [str(idx) for idx in crosstab.index], + "columns": [str(col) for col in crosstab.columns], + "counts": counts.astype(int).tolist(), + }, + }, + }, + } diff --git a/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py b/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py index dcf85420f954..70e8927c6a71 100644 --- a/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py +++ b/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py @@ -1,27 +1,346 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union + +from great_expectations.compatibility import pydantic +from great_expectations.compatibility.typing_extensions import override +from great_expectations.core.suite_parameters import ( + SuiteParameterDict, # noqa: TC001 # FIXME CoP +) from great_expectations.expectations.expectation import ( - BatchExpectation, + ColumnAggregateExpectation, + _style_row_condition, + render_suite_parameter_string, +) +from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources +from great_expectations.expectations.model_field_descriptions import ( + COLUMN_DESCRIPTION, + FAILURE_SEVERITY_DESCRIPTION, +) +from great_expectations.render import ( + LegacyRendererType, + RenderedStringTemplateContent, +) +from great_expectations.render.renderer.renderer import renderer +from great_expectations.render.renderer_configuration import ( + RendererConfiguration, + RendererValueType, +) +from great_expectations.render.util import ( + parse_row_condition_string, + substitute_none_for_missing, +) + +if TYPE_CHECKING: + from great_expectations.core import ( + ExpectationValidationResult, + ) + from great_expectations.execution_engine import ExecutionEngine + from great_expectations.expectations.expectation_configuration import ( + ExpectationConfiguration, + ) + from great_expectations.render.renderer_configuration import AddParamArgs + +EXPECTATION_SHORT_DESCRIPTION = ( + "Expect the p-value of a one-sample Kolmogorov-Smirnov test comparing the column to a " + "named, parameterized theoretical distribution to be greater than or equal to a threshold." +) +DISTRIBUTION_DESCRIPTION = ( + "The name of the scipy.stats continuous distribution to test against " + "(e.g. 'norm', 'expon', 'beta', 'gamma', 'uniform', 'chi2', 'lognorm')." ) +P_VALUE_DESCRIPTION = ( + "The threshold p-value. The Expectation succeeds when the observed Kolmogorov-Smirnov " + "p-value is greater than or equal to this value. Must be strictly between 0 and 1. Defaults to 0.05." # noqa: E501 # FIXME CoP +) +PARAMS_DESCRIPTION = ( + "The parameters of the theoretical distribution, supplied either positionally as a list " + "or by name as a dict (e.g. {'mean': 0, 'std_dev': 1} for 'norm')." +) +SUPPORTED_DATA_SOURCES = [ + SupportedDataSources.PANDAS.value, +] +DATA_QUALITY_ISSUES = [DataQualityIssues.NUMERIC.value] + + +class ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan(ColumnAggregateExpectation): + __doc__ = f"""{EXPECTATION_SHORT_DESCRIPTION} + + ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan is a \ + Column Aggregate Expectation. + + Column Aggregate Expectations are one of the most common types of Expectation. + They are evaluated for a single column, and produce an aggregate Metric, such as a mean, standard deviation, number of unique values, column type, etc. + If that Metric meets the conditions you set, the Expectation considers that data valid. + + The Kolmogorov-Smirnov test compares the empirical distribution of the column with the \ + cumulative distribution function of the named theoretical distribution. A large p-value \ + indicates that the data are consistent with having been drawn from that distribution. + + Args: + column (str): \ + {COLUMN_DESCRIPTION} + distribution (str): \ + {DISTRIBUTION_DESCRIPTION} + p_value (float): \ + {P_VALUE_DESCRIPTION} + params (list or dict or None): \ + {PARAMS_DESCRIPTION} + + Other Parameters: + result_format (str or None): \ + Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ + For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). + catch_exceptions (boolean or None): \ + If True, then catch exceptions and include them as part of the result object. \ + For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions). + meta (dict or None): \ + A JSON-serializable dictionary (nesting allowed) that will be included in the output without \ + modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta). + severity (str or None): \ + {FAILURE_SEVERITY_DESCRIPTION} \ + For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity). + + Returns: + An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result) + Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta. -# NOTE: This Expectation is incomplete and not ready for use. -# It should remain unexported until it meets the requirements set by our V1 API. -class ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan(BatchExpectation): - def __init__(self, *args, **kwargs): - raise NotImplementedError + Notes: + * observed_value field in the result object is customized for this expectation to be the \ + p-value of the Kolmogorov-Smirnov test. + * The Expectation succeeds when the observed p-value is greater than or equal to p_value. - library_metadata = { + Supported Data Sources: + [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) + + Data Quality Issues: + {DATA_QUALITY_ISSUES[0]} + + Example Data: + test + 0 0.1 + 1 -0.2 + 2 0.4 + 3 -0.5 + 4 0.3 + + Code Examples: + Passing Case: + Input: + ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan( + column="test", + distribution="norm", + p_value=0.05, + params={{"mean": 0, "std_dev": 1}} + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 0.9 + }}, + "meta": {{}}, + "success": true + }} + + Failing Case: + Input: + ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan( + column="test", + distribution="expon", + p_value=0.05 + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "observed_value": 0.0001 + }}, + "meta": {{}}, + "success": false + }} + """ # noqa: E501 # FIXME CoP + + distribution: str = pydantic.Field(description=DISTRIBUTION_DESCRIPTION) + p_value: Union[float, SuiteParameterDict] = pydantic.Field( + default=0.05, description=P_VALUE_DESCRIPTION + ) + params: Union[List[float], Dict[str, float], SuiteParameterDict, None] = pydantic.Field( + default=None, description=PARAMS_DESCRIPTION + ) + + library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", "column aggregate expectation", - "needs migration to modular expectations api", + "distributional expectation", ], "contributors": ["@great_expectations"], "requirements": [], + "has_full_test_suite": True, + "manually_reviewed_code": True, } + _library_metadata = library_metadata + + metric_dependencies = ("column.parameterized_distribution_ks_test_p_value",) + success_keys = ( + "distribution", + "p_value", + "params", + ) + args_keys = ( + "column", + "distribution", + "p_value", + "params", + ) + + class Config: + title = "Expect column parameterized distribution KS test p-value to be greater than" + + @staticmethod + def schema_extra( + schema: Dict[str, Any], + model: Type[ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan], + ) -> None: + ColumnAggregateExpectation.Config.schema_extra(schema, model) + schema["properties"]["metadata"]["properties"].update( + { + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": DATA_QUALITY_ISSUES, + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": model._library_metadata, + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": EXPECTATION_SHORT_DESCRIPTION, + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": SUPPORTED_DATA_SOURCES, + }, + } + ) + + @classmethod + @override + def _prescriptive_template( + cls, + renderer_configuration: RendererConfiguration, + ) -> RendererConfiguration: + add_param_args: AddParamArgs = ( + ("column", RendererValueType.STRING), + ("distribution", RendererValueType.STRING), + ("p_value", RendererValueType.NUMBER), + ) + for name, param_type in add_param_args: + renderer_configuration.add_param(name=name, param_type=param_type) + + template_str = ( + "Kolmogorov-Smirnov test p-value against the $distribution distribution " + "must be greater than or equal to $p_value." + ) + if renderer_configuration.include_column_name: + template_str = f"$column {template_str}" + + renderer_configuration.template_str = template_str + return renderer_configuration + + @classmethod + @override + @renderer(renderer_type=LegacyRendererType.PRESCRIPTIVE) + @render_suite_parameter_string + def _prescriptive_renderer( # type: ignore[override] # TODO: Fix this type ignore + cls, + configuration: ExpectationConfiguration, + result: Optional[ExpectationValidationResult] = None, + runtime_configuration: Optional[dict] = None, + **kwargs, + ): + runtime_configuration = runtime_configuration or {} + include_column_name = runtime_configuration.get("include_column_name") is not False + styling = runtime_configuration.get("styling") + params = substitute_none_for_missing( + configuration.kwargs, + [ + "column", + "distribution", + "p_value", + "row_condition", + "condition_parser", + ], + ) + + template_str = ( + "Kolmogorov-Smirnov test p-value against the $distribution distribution " + "must be greater than or equal to $p_value." + ) + if include_column_name: + template_str = f"$column {template_str}" + + if params["row_condition"] is not None: + conditional_template_str = parse_row_condition_string(params["row_condition"]) + template_str, styling = _style_row_condition( + conditional_template_str, + template_str, + params, + styling, + ) + + return [ + RenderedStringTemplateContent( + content_block_type="string_template", + string_template={ + "template": template_str, + "params": params, + "styling": styling, + }, + ) + ] + + @override + def _validate( + self, + metrics: Dict, + runtime_configuration: Optional[dict] = None, + execution_engine: Optional[ExecutionEngine] = None, + ): + configuration = self.configuration + p_value = configuration.kwargs.get("p_value", self._get_default_value("p_value")) + + ks_result = metrics["column.parameterized_distribution_ks_test_p_value"] + # scipy returns a KstestResult (statistic, pvalue) named tuple. + observed_statistic = float(ks_result[0]) + observed_p_value = float(ks_result[1]) - metric_dependencies = tuple() - success_keys = () - args_keys = () + return { + "success": bool(observed_p_value >= p_value), + "result": { + "observed_value": observed_p_value, + "details": { + "observed_ks_result": { + "statistic": observed_statistic, + "pvalue": observed_p_value, + }, + }, + }, + } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnBootstrappedKsTestPValueToBeGreaterThan.json b/great_expectations/expectations/core/schemas/ExpectColumnBootstrappedKsTestPValueToBeGreaterThan.json new file mode 100644 index 000000000000..d436084ad54a --- /dev/null +++ b/great_expectations/expectations/core/schemas/ExpectColumnBootstrappedKsTestPValueToBeGreaterThan.json @@ -0,0 +1,462 @@ +{ + "title": "Expect column bootstrapped KS test p-value to be greater than", + "description": "Expect the bootstrapped Kolmogorov-Smirnov test p-value statistic comparing the column to a continuous partition object to be greater than a threshold.\n\nExpectColumnBootstrappedKsTestPValueToBeGreaterThan is a Column Aggregate Expectation.\n\nColumn Aggregate Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column, and produce an aggregate Metric, such as a mean, standard deviation, number of unique values, column type, etc.\nIf that Metric meets the conditions you set, the Expectation considers that data valid.\n\nThis Expectation repeatedly draws bootstrap samples from the column and runs a one-sample Kolmogorov-Smirnov test against the cumulative distribution function implied by the provided continuous partition object. The reported statistic is the fraction of bootstrap rounds whose KS p-value is at least ``p``; a high value indicates the data are consistent with the partition.\n\nArgs:\n column (str): The column name.\n partition_object (dict): The expected continuous partition object, with finite ``bins`` and normalized ``weights`` and no tail weights. See [partition_object](https://docs.greatexpectations.io/docs/reference/expectations/distributional_expectations/#partition-objects).\n p (float): The threshold below which a p-value counts as a failure. The Expectation succeeds when the bootstrapped statistic is greater than this value. Defaults to 0.05.\n bootstrap_samples (int or None): The number of bootstrap rounds to perform. Defaults to 1000 when not provided.\n bootstrap_sample_size (int or None): The number of elements to draw (with replacement) in each bootstrap round. Defaults to twice the number of partition weights when not provided.\n\nOther Parameters:\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nNotes:\n * observed_value field in the result object is customized for this expectation to be the bootstrapped p-value statistic (the fraction of bootstrap rounds whose KS p-value was at least ``p``).\n * The Expectation succeeds when the observed statistic is greater than ``p``.\n * Because the statistic is computed from random bootstrap samples, the observed_value may vary slightly between runs.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n test\n 0 0.1\n 1 0.4\n 2 0.6\n 3 0.9\n 4 0.5\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnBootstrappedKsTestPValueToBeGreaterThan(\n column=\"test\",\n partition_object={\n \"bins\": [0.0, 0.25, 0.5, 0.75, 1.0],\n \"weights\": [0.25, 0.25, 0.25, 0.25],\n },\n p=0.05\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.8\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnBootstrappedKsTestPValueToBeGreaterThan(\n column=\"test\",\n partition_object={\n \"bins\": [0.0, 0.25, 0.5, 0.75, 1.0],\n \"weights\": [0.97, 0.01, 0.01, 0.01],\n },\n p=0.05\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.0\n },\n \"meta\": {},\n \"success\": false\n }", + "type": "object", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "meta": { + "title": "Meta", + "type": "object" + }, + "notes": { + "title": "Notes", + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "result_format": { + "title": "Result Format", + "default": "BASIC", + "anyOf": [ + { + "$ref": "#/definitions/ResultFormat" + }, + { + "type": "object" + } + ] + }, + "description": { + "title": "Description", + "description": "A short description of your Expectation", + "type": "string" + }, + "catch_exceptions": { + "title": "Catch Exceptions", + "default": false, + "type": "boolean" + }, + "rendered_content": { + "title": "Rendered Content", + "type": "array", + "items": { + "type": "object" + } + }, + "severity": { + "description": "Indicate the impact of this Expectation failing. Severity levels can be used to trigger different alerting patterns and actions.", + "default": "critical", + "allOf": [ + { + "$ref": "#/definitions/FailureSeverity" + } + ] + }, + "windows": { + "title": "Windows", + "description": "Definition(s) for evaluation of temporal windows", + "type": "array", + "items": { + "$ref": "#/definitions/Window" + } + }, + "batch_id": { + "title": "Batch Id", + "type": "string" + }, + "column": { + "title": "Column", + "description": "The column name.", + "minLength": 1, + "type": "string" + }, + "row_condition": { + "title": "Row Condition", + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ComparisonCondition" + }, + { + "$ref": "#/definitions/NullityCondition" + }, + { + "$ref": "#/definitions/AndCondition" + }, + { + "$ref": "#/definitions/OrCondition" + }, + { + "$ref": "#/definitions/PassThroughCondition" + } + ] + }, + "condition_parser": { + "title": "Condition Parser", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" + }, + "partition_object": { + "title": "Partition Object", + "description": "The expected continuous partition object, with finite ``bins`` and normalized ``weights`` and no tail weights.", + "type": "object" + }, + "p": { + "title": "P", + "description": "The threshold below which a p-value counts as a failure. The Expectation succeeds when the bootstrapped statistic is greater than this value. Defaults to 0.05.", + "default": 0.05, + "anyOf": [ + { + "type": "number" + }, + { + "type": "object" + } + ] + }, + "bootstrap_samples": { + "title": "Bootstrap Samples", + "description": "The number of bootstrap rounds to perform. Defaults to 1000 when not provided.", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "object" + } + ] + }, + "bootstrap_sample_size": { + "title": "Bootstrap Sample Size", + "description": "The number of elements to draw (with replacement) in each bootstrap round. Defaults to twice the number of partition weights when not provided.", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "object" + } + ] + }, + "metadata": { + "type": "object", + "properties": { + "expectation_class": { + "title": "Expectation Class", + "type": "string", + "const": "ExpectColumnBootstrappedKsTestPValueToBeGreaterThan" + }, + "expectation_type": { + "title": "Expectation Type", + "type": "string", + "const": "expect_column_bootstrapped_ks_test_p_value_to_be_greater_than" + }, + "domain_type": { + "title": "Domain Type", + "type": "string", + "const": "column", + "description": "Column Aggregate" + }, + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": [ + "Numeric" + ] + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": { + "maturity": "production", + "tags": [ + "core expectation", + "column aggregate expectation", + "distributional expectation" + ], + "contributors": [ + "@great_expectations" + ], + "requirements": [], + "has_full_test_suite": true, + "manually_reviewed_code": true + } + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": "Expect the bootstrapped Kolmogorov-Smirnov test p-value statistic comparing the column to a continuous partition object to be greater than a threshold." + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": [ + "Pandas" + ] + } + } + } + }, + "required": [ + "column", + "partition_object" + ], + "additionalProperties": false, + "definitions": { + "ResultFormat": { + "title": "ResultFormat", + "description": "An enumeration.", + "enum": [ + "BOOLEAN_ONLY", + "BASIC", + "COMPLETE", + "SUMMARY" + ], + "type": "string" + }, + "FailureSeverity": { + "title": "FailureSeverity", + "description": "Severity levels for Expectation failures.", + "enum": [ + "critical", + "warning", + "info" + ], + "type": "string" + }, + "Offset": { + "title": "Offset", + "description": "A threshold in which a metric will be considered passable", + "type": "object", + "properties": { + "positive": { + "title": "Positive", + "type": "number" + }, + "negative": { + "title": "Negative", + "type": "number" + } + }, + "required": [ + "positive", + "negative" + ], + "additionalProperties": false + }, + "Window": { + "title": "Window", + "description": "A definition for a temporal window across <`range`> number of previous invocations", + "type": "object", + "properties": { + "constraint_fn": { + "title": "Constraint Fn", + "type": "string" + }, + "parameter_name": { + "title": "Parameter Name", + "type": "string" + }, + "range": { + "title": "Range", + "type": "integer" + }, + "offset": { + "$ref": "#/definitions/Offset" + }, + "strict": { + "title": "Strict", + "default": false, + "type": "boolean" + } + }, + "required": [ + "constraint_fn", + "parameter_name", + "range", + "offset" + ], + "additionalProperties": false + }, + "Column": { + "title": "Column", + "description": "--Public API--\nSpecify the column in a condition statement.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "Operator": { + "title": "Operator", + "description": "An enumeration.", + "enum": [ + "==", + "!=", + "<", + "<=", + ">", + ">=", + "IN", + "NOT_IN" + ], + "type": "string" + }, + "ComparisonCondition": { + "title": "ComparisonCondition", + "description": "--Public API--Condition representing the comparison of a column with a parameter.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "comparison", + "enum": [ + "comparison" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "operator": { + "$ref": "#/definitions/Operator" + }, + "parameter": { + "title": "Parameter" + } + }, + "required": [ + "column", + "operator", + "parameter" + ] + }, + "NullityCondition": { + "title": "NullityCondition", + "description": "--Public API--Condition representing the whether or not a column is null.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "nullity", + "enum": [ + "nullity" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "is_null": { + "title": "Is Null", + "type": "boolean" + } + }, + "required": [ + "column", + "is_null" + ] + }, + "Condition": { + "title": "Condition", + "description": "Base class for conditions.", + "type": "object", + "properties": {} + }, + "AndCondition": { + "title": "AndCondition", + "description": "--Public API--Represents an AND condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "and", + "enum": [ + "and" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "OrCondition": { + "title": "OrCondition", + "description": "--Public API--Represents an OR condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "or", + "enum": [ + "or" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "PassThroughCondition": { + "title": "PassThroughCondition", + "description": "Condition that passes a filter string directly to the execution engine.\n\nThis is used for legacy pandas/spark condition_parser syntax where the\nrow_condition string is passed directly to DataFrame.query() or DataFrame.filter().", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "pass_through", + "enum": [ + "pass_through" + ], + "type": "string" + }, + "pass_through_filter": { + "title": "Pass Through Filter", + "type": "string" + } + }, + "required": [ + "pass_through_filter" + ] + } + } +} diff --git a/great_expectations/expectations/core/schemas/ExpectColumnChisquareTestPValueToBeGreaterThan.json b/great_expectations/expectations/core/schemas/ExpectColumnChisquareTestPValueToBeGreaterThan.json new file mode 100644 index 000000000000..1fc14e5bd018 --- /dev/null +++ b/great_expectations/expectations/core/schemas/ExpectColumnChisquareTestPValueToBeGreaterThan.json @@ -0,0 +1,453 @@ +{ + "title": "Expect column Chi-square test p-value to be greater than", + "description": "Expect the p-value of a Chi-square goodness-of-fit test comparing the observed categorical frequencies of the column to an expected partition object to be greater than a threshold.\n\nExpectColumnChisquareTestPValueToBeGreaterThan is a Column Aggregate Expectation.\n\nColumn Aggregate Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column, and produce an aggregate Metric, such as a mean, standard deviation, number of unique values, column type, etc.\nIf that Metric meets the conditions you set, the Expectation considers that data valid.\n\nThe Chi-square goodness-of-fit test compares the observed value counts of the column with the expected counts implied by the partition object. A large p-value indicates that the observed distribution is consistent with the expected one.\n\nArgs:\n column (str): The column name.\n partition_object (dict): The expected categorical partition object, with ``values`` and normalized ``weights``. See [partition_object](https://docs.greatexpectations.io/docs/reference/expectations/distributional_expectations/#partition-objects).\n p (float): The threshold p-value. The Expectation succeeds when the observed Chi-square p-value is greater than this value. Defaults to 0.05.\n tail_weight_holdout (float): The amount of weight to split uniformly among values observed in the data but absent from the partition object. Provides a mechanism to make the test less strict. Defaults to 0.\n\nOther Parameters:\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nNotes:\n * observed_value field in the result object is customized for this expectation to be the p-value of the Chi-square test.\n * details.observed_partition and details.expected_partition are customized for this expectation to be dicts representing the observed and expected partitions.\n * The Expectation succeeds when the observed p-value is greater than ``p``.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n test\n 0 \"A\"\n 1 \"A\"\n 2 \"B\"\n 3 \"B\"\n 4 \"C\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnChisquareTestPValueToBeGreaterThan(\n column=\"test\",\n partition_object={\"values\": [\"A\", \"B\", \"C\"], \"weights\": [0.4, 0.4, 0.2]},\n p=0.05\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 1.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnChisquareTestPValueToBeGreaterThan(\n column=\"test\",\n partition_object={\"values\": [\"A\", \"B\", \"C\"], \"weights\": [0.05, 0.05, 0.9]},\n p=0.05\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.0001\n },\n \"meta\": {},\n \"success\": false\n }", + "type": "object", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "meta": { + "title": "Meta", + "type": "object" + }, + "notes": { + "title": "Notes", + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "result_format": { + "title": "Result Format", + "default": "BASIC", + "anyOf": [ + { + "$ref": "#/definitions/ResultFormat" + }, + { + "type": "object" + } + ] + }, + "description": { + "title": "Description", + "description": "A short description of your Expectation", + "type": "string" + }, + "catch_exceptions": { + "title": "Catch Exceptions", + "default": false, + "type": "boolean" + }, + "rendered_content": { + "title": "Rendered Content", + "type": "array", + "items": { + "type": "object" + } + }, + "severity": { + "description": "Indicate the impact of this Expectation failing. Severity levels can be used to trigger different alerting patterns and actions.", + "default": "critical", + "allOf": [ + { + "$ref": "#/definitions/FailureSeverity" + } + ] + }, + "windows": { + "title": "Windows", + "description": "Definition(s) for evaluation of temporal windows", + "type": "array", + "items": { + "$ref": "#/definitions/Window" + } + }, + "batch_id": { + "title": "Batch Id", + "type": "string" + }, + "column": { + "title": "Column", + "description": "The column name.", + "minLength": 1, + "type": "string" + }, + "row_condition": { + "title": "Row Condition", + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ComparisonCondition" + }, + { + "$ref": "#/definitions/NullityCondition" + }, + { + "$ref": "#/definitions/AndCondition" + }, + { + "$ref": "#/definitions/OrCondition" + }, + { + "$ref": "#/definitions/PassThroughCondition" + } + ] + }, + "condition_parser": { + "title": "Condition Parser", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" + }, + "partition_object": { + "title": "Partition Object", + "description": "The expected categorical partition object, with ``values`` and normalized ``weights``.", + "type": "object" + }, + "p": { + "title": "P", + "description": "The threshold p-value. The Expectation succeeds when the observed Chi-square p-value is greater than this value. Defaults to 0.05.", + "default": 0.05, + "anyOf": [ + { + "type": "number" + }, + { + "type": "object" + } + ] + }, + "tail_weight_holdout": { + "title": "Tail Weight Holdout", + "description": "The amount of weight to split uniformly among values observed in the data but absent from the partition object. Provides a mechanism to make the test less strict. Defaults to 0.", + "default": 0, + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "object" + } + ] + }, + "metadata": { + "type": "object", + "properties": { + "expectation_class": { + "title": "Expectation Class", + "type": "string", + "const": "ExpectColumnChisquareTestPValueToBeGreaterThan" + }, + "expectation_type": { + "title": "Expectation Type", + "type": "string", + "const": "expect_column_chisquare_test_p_value_to_be_greater_than" + }, + "domain_type": { + "title": "Domain Type", + "type": "string", + "const": "column", + "description": "Column Aggregate" + }, + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": [ + "Numeric" + ] + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": { + "maturity": "production", + "tags": [ + "core expectation", + "column aggregate expectation", + "distributional expectation" + ], + "contributors": [ + "@great_expectations" + ], + "requirements": [], + "has_full_test_suite": true, + "manually_reviewed_code": true + } + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": "Expect the p-value of a Chi-square goodness-of-fit test comparing the observed categorical frequencies of the column to an expected partition object to be greater than a threshold." + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": [ + "Pandas" + ] + } + } + } + }, + "required": [ + "column", + "partition_object" + ], + "additionalProperties": false, + "definitions": { + "ResultFormat": { + "title": "ResultFormat", + "description": "An enumeration.", + "enum": [ + "BOOLEAN_ONLY", + "BASIC", + "COMPLETE", + "SUMMARY" + ], + "type": "string" + }, + "FailureSeverity": { + "title": "FailureSeverity", + "description": "Severity levels for Expectation failures.", + "enum": [ + "critical", + "warning", + "info" + ], + "type": "string" + }, + "Offset": { + "title": "Offset", + "description": "A threshold in which a metric will be considered passable", + "type": "object", + "properties": { + "positive": { + "title": "Positive", + "type": "number" + }, + "negative": { + "title": "Negative", + "type": "number" + } + }, + "required": [ + "positive", + "negative" + ], + "additionalProperties": false + }, + "Window": { + "title": "Window", + "description": "A definition for a temporal window across <`range`> number of previous invocations", + "type": "object", + "properties": { + "constraint_fn": { + "title": "Constraint Fn", + "type": "string" + }, + "parameter_name": { + "title": "Parameter Name", + "type": "string" + }, + "range": { + "title": "Range", + "type": "integer" + }, + "offset": { + "$ref": "#/definitions/Offset" + }, + "strict": { + "title": "Strict", + "default": false, + "type": "boolean" + } + }, + "required": [ + "constraint_fn", + "parameter_name", + "range", + "offset" + ], + "additionalProperties": false + }, + "Column": { + "title": "Column", + "description": "--Public API--\nSpecify the column in a condition statement.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "Operator": { + "title": "Operator", + "description": "An enumeration.", + "enum": [ + "==", + "!=", + "<", + "<=", + ">", + ">=", + "IN", + "NOT_IN" + ], + "type": "string" + }, + "ComparisonCondition": { + "title": "ComparisonCondition", + "description": "--Public API--Condition representing the comparison of a column with a parameter.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "comparison", + "enum": [ + "comparison" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "operator": { + "$ref": "#/definitions/Operator" + }, + "parameter": { + "title": "Parameter" + } + }, + "required": [ + "column", + "operator", + "parameter" + ] + }, + "NullityCondition": { + "title": "NullityCondition", + "description": "--Public API--Condition representing the whether or not a column is null.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "nullity", + "enum": [ + "nullity" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "is_null": { + "title": "Is Null", + "type": "boolean" + } + }, + "required": [ + "column", + "is_null" + ] + }, + "Condition": { + "title": "Condition", + "description": "Base class for conditions.", + "type": "object", + "properties": {} + }, + "AndCondition": { + "title": "AndCondition", + "description": "--Public API--Represents an AND condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "and", + "enum": [ + "and" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "OrCondition": { + "title": "OrCondition", + "description": "--Public API--Represents an OR condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "or", + "enum": [ + "or" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "PassThroughCondition": { + "title": "PassThroughCondition", + "description": "Condition that passes a filter string directly to the execution engine.\n\nThis is used for legacy pandas/spark condition_parser syntax where the\nrow_condition string is passed directly to DataFrame.query() or DataFrame.filter().", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "pass_through", + "enum": [ + "pass_through" + ], + "type": "string" + }, + "pass_through_filter": { + "title": "Pass Through Filter", + "type": "string" + } + }, + "required": [ + "pass_through_filter" + ] + } + } +} diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json new file mode 100644 index 000000000000..f3c2e5f17e79 --- /dev/null +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json @@ -0,0 +1,266 @@ +{ + "title": "Expect column pair Cram\u00e9r's phi value to be less than", + "description": "Expect the Cram\u00e9r's phi (V) measure of association between two columns to be less than or equal to a threshold.\n\nExpectColumnPairCramersPhiValueToBeLessThan is a Batch Expectation.\n\nBatchExpectations are one of the most common types of Expectation.\nThey are evaluated for an entire Batch, and answer a semantic question about the Batch itself.\n\nCram\u00e9r's phi (also written Cram\u00e9r's V) measures the strength of association between two categorical variables, derived from the Chi-square statistic of their contingency table. A value near 0 indicates the columns are (close to) independent; larger values indicate stronger association. Numeric columns are discretized into bins before the table is built.\n\nArgs:\n column_A (str): The first column name.\n column_B (str): The second column name.\n threshold (float): The maximum Cram\u00e9r's phi value for which to return success=True. Cram\u00e9r's phi ranges from 0 (no association / independent) to 1 (perfect association). Defaults to 0.1.\n bins_A (list or None): Explicit bin edges (numeric column) or groups of values (categorical column) used to discretize column_A before building the contingency table.\n bins_B (list or None): As bins_A, but for column_B.\n n_bins_A (int or None): The number of bins to use for column_A when bins_A is not provided. Defaults to 10.\n n_bins_B (int or None): As n_bins_A, but for column_B.\n\nOther Parameters:\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nNotes:\n * observed_value field in the result object is customized for this expectation to be the Cram\u00e9r's phi (V) value.\n * details.crosstab is customized for this expectation to be a serializable representation of the contingency table between column_A and column_B.\n * The Expectation succeeds when the observed Cram\u00e9r's phi is less than or equal to threshold.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n test test2\n 0 \"A\" \"X\"\n 1 \"A\" \"Y\"\n 2 \"B\" \"X\"\n 3 \"B\" \"Y\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnPairCramersPhiValueToBeLessThan(\n column_A=\"test\",\n column_B=\"test2\",\n threshold=0.1\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnPairCramersPhiValueToBeLessThan(\n column_A=\"test\",\n column_B=\"test2\",\n threshold=0.1\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 1.0\n },\n \"meta\": {},\n \"success\": false\n }", + "type": "object", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "meta": { + "title": "Meta", + "type": "object" + }, + "notes": { + "title": "Notes", + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "result_format": { + "title": "Result Format", + "default": "BASIC", + "anyOf": [ + { + "$ref": "#/definitions/ResultFormat" + }, + { + "type": "object" + } + ] + }, + "description": { + "title": "Description", + "description": "A short description of your Expectation", + "type": "string" + }, + "catch_exceptions": { + "title": "Catch Exceptions", + "default": false, + "type": "boolean" + }, + "rendered_content": { + "title": "Rendered Content", + "type": "array", + "items": { + "type": "object" + } + }, + "severity": { + "description": "Indicate the impact of this Expectation failing. Severity levels can be used to trigger different alerting patterns and actions.", + "default": "critical", + "allOf": [ + { + "$ref": "#/definitions/FailureSeverity" + } + ] + }, + "windows": { + "title": "Windows", + "description": "Definition(s) for evaluation of temporal windows", + "type": "array", + "items": { + "$ref": "#/definitions/Window" + } + }, + "batch_id": { + "title": "Batch Id", + "type": "string" + }, + "column_A": { + "title": "Column A", + "description": "The first column name.", + "minLength": 1, + "type": "string" + }, + "column_B": { + "title": "Column B", + "description": "The second column name.", + "minLength": 1, + "type": "string" + }, + "threshold": { + "title": "Threshold", + "description": "The maximum Cram\u00e9r's phi value for which to return success=True. Cram\u00e9r's phi ranges from 0 (no association / independent) to 1 (perfect association). Defaults to 0.1.", + "default": 0.1, + "anyOf": [ + { + "type": "number" + }, + { + "type": "object" + } + ] + }, + "bins_A": { + "title": "Bins A", + "description": "Explicit bin edges (numeric column) or groups of values (categorical column) used to discretize column_A before building the contingency table.", + "type": "array", + "items": {} + }, + "bins_B": { + "title": "Bins B", + "description": "As bins_A, but for column_B.", + "type": "array", + "items": {} + }, + "n_bins_A": { + "title": "N Bins A", + "description": "The number of bins to use for column_A when bins_A is not provided. Defaults to 10.", + "type": "integer" + }, + "n_bins_B": { + "title": "N Bins B", + "description": "As n_bins_A, but for column_B.", + "type": "integer" + }, + "metadata": { + "type": "object", + "properties": { + "expectation_class": { + "title": "Expectation Class", + "type": "string", + "const": "ExpectColumnPairCramersPhiValueToBeLessThan" + }, + "expectation_type": { + "title": "Expectation Type", + "type": "string", + "const": "expect_column_pair_cramers_phi_value_to_be_less_than" + }, + "domain_type": { + "title": "Domain Type", + "type": "string", + "const": "table", + "description": "Batch" + }, + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": [ + "Numeric" + ] + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": { + "maturity": "production", + "tags": [ + "core expectation", + "multi-column expectation", + "distributional expectation" + ], + "contributors": [ + "@great_expectations" + ], + "requirements": [], + "has_full_test_suite": true, + "manually_reviewed_code": true + } + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": "Expect the Cram\u00e9r's phi (V) measure of association between two columns to be less than or equal to a threshold." + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": [ + "Pandas" + ] + } + } + } + }, + "required": [ + "column_A", + "column_B" + ], + "additionalProperties": false, + "definitions": { + "ResultFormat": { + "title": "ResultFormat", + "description": "An enumeration.", + "enum": [ + "BOOLEAN_ONLY", + "BASIC", + "COMPLETE", + "SUMMARY" + ], + "type": "string" + }, + "FailureSeverity": { + "title": "FailureSeverity", + "description": "Severity levels for Expectation failures.", + "enum": [ + "critical", + "warning", + "info" + ], + "type": "string" + }, + "Offset": { + "title": "Offset", + "description": "A threshold in which a metric will be considered passable", + "type": "object", + "properties": { + "positive": { + "title": "Positive", + "type": "number" + }, + "negative": { + "title": "Negative", + "type": "number" + } + }, + "required": [ + "positive", + "negative" + ], + "additionalProperties": false + }, + "Window": { + "title": "Window", + "description": "A definition for a temporal window across <`range`> number of previous invocations", + "type": "object", + "properties": { + "constraint_fn": { + "title": "Constraint Fn", + "type": "string" + }, + "parameter_name": { + "title": "Parameter Name", + "type": "string" + }, + "range": { + "title": "Range", + "type": "integer" + }, + "offset": { + "$ref": "#/definitions/Offset" + }, + "strict": { + "title": "Strict", + "default": false, + "type": "boolean" + } + }, + "required": [ + "constraint_fn", + "parameter_name", + "range", + "offset" + ], + "additionalProperties": false + } + } +} diff --git a/great_expectations/expectations/core/schemas/ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan.json b/great_expectations/expectations/core/schemas/ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan.json new file mode 100644 index 000000000000..df471c6fea2f --- /dev/null +++ b/great_expectations/expectations/core/schemas/ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan.json @@ -0,0 +1,459 @@ +{ + "title": "Expect column parameterized distribution KS test p-value to be greater than", + "description": "Expect the p-value of a one-sample Kolmogorov-Smirnov test comparing the column to a named, parameterized theoretical distribution to be greater than or equal to a threshold.\n\nExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan is a Column Aggregate Expectation.\n\nColumn Aggregate Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column, and produce an aggregate Metric, such as a mean, standard deviation, number of unique values, column type, etc.\nIf that Metric meets the conditions you set, the Expectation considers that data valid.\n\nThe Kolmogorov-Smirnov test compares the empirical distribution of the column with the cumulative distribution function of the named theoretical distribution. A large p-value indicates that the data are consistent with having been drawn from that distribution.\n\nArgs:\n column (str): The column name.\n distribution (str): The name of the scipy.stats continuous distribution to test against (e.g. 'norm', 'expon', 'beta', 'gamma', 'uniform', 'chi2', 'lognorm').\n p_value (float): The threshold p-value. The Expectation succeeds when the observed Kolmogorov-Smirnov p-value is greater than or equal to this value. Must be strictly between 0 and 1. Defaults to 0.05.\n params (list or dict or None): The parameters of the theoretical distribution, supplied either positionally as a list or by name as a dict (e.g. {'mean': 0, 'std_dev': 1} for 'norm').\n\nOther Parameters:\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nNotes:\n * observed_value field in the result object is customized for this expectation to be the p-value of the Kolmogorov-Smirnov test.\n * The Expectation succeeds when the observed p-value is greater than or equal to p_value.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n test\n 0 0.1\n 1 -0.2\n 2 0.4\n 3 -0.5\n 4 0.3\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan(\n column=\"test\",\n distribution=\"norm\",\n p_value=0.05,\n params={\"mean\": 0, \"std_dev\": 1}\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.9\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan(\n column=\"test\",\n distribution=\"expon\",\n p_value=0.05\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.0001\n },\n \"meta\": {},\n \"success\": false\n }", + "type": "object", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "meta": { + "title": "Meta", + "type": "object" + }, + "notes": { + "title": "Notes", + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "result_format": { + "title": "Result Format", + "default": "BASIC", + "anyOf": [ + { + "$ref": "#/definitions/ResultFormat" + }, + { + "type": "object" + } + ] + }, + "description": { + "title": "Description", + "description": "A short description of your Expectation", + "type": "string" + }, + "catch_exceptions": { + "title": "Catch Exceptions", + "default": false, + "type": "boolean" + }, + "rendered_content": { + "title": "Rendered Content", + "type": "array", + "items": { + "type": "object" + } + }, + "severity": { + "description": "Indicate the impact of this Expectation failing. Severity levels can be used to trigger different alerting patterns and actions.", + "default": "critical", + "allOf": [ + { + "$ref": "#/definitions/FailureSeverity" + } + ] + }, + "windows": { + "title": "Windows", + "description": "Definition(s) for evaluation of temporal windows", + "type": "array", + "items": { + "$ref": "#/definitions/Window" + } + }, + "batch_id": { + "title": "Batch Id", + "type": "string" + }, + "column": { + "title": "Column", + "description": "The column name.", + "minLength": 1, + "type": "string" + }, + "row_condition": { + "title": "Row Condition", + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ComparisonCondition" + }, + { + "$ref": "#/definitions/NullityCondition" + }, + { + "$ref": "#/definitions/AndCondition" + }, + { + "$ref": "#/definitions/OrCondition" + }, + { + "$ref": "#/definitions/PassThroughCondition" + } + ] + }, + "condition_parser": { + "title": "Condition Parser", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" + }, + "distribution": { + "title": "Distribution", + "description": "The name of the scipy.stats continuous distribution to test against (e.g. 'norm', 'expon', 'beta', 'gamma', 'uniform', 'chi2', 'lognorm').", + "type": "string" + }, + "p_value": { + "title": "P Value", + "description": "The threshold p-value. The Expectation succeeds when the observed Kolmogorov-Smirnov p-value is greater than or equal to this value. Must be strictly between 0 and 1. Defaults to 0.05.", + "default": 0.05, + "anyOf": [ + { + "type": "number" + }, + { + "type": "object" + } + ] + }, + "params": { + "title": "Params", + "description": "The parameters of the theoretical distribution, supplied either positionally as a list or by name as a dict (e.g. {'mean': 0, 'std_dev': 1} for 'norm').", + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + { + "type": "object" + } + ] + }, + "metadata": { + "type": "object", + "properties": { + "expectation_class": { + "title": "Expectation Class", + "type": "string", + "const": "ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan" + }, + "expectation_type": { + "title": "Expectation Type", + "type": "string", + "const": "expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than" + }, + "domain_type": { + "title": "Domain Type", + "type": "string", + "const": "column", + "description": "Column Aggregate" + }, + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": [ + "Numeric" + ] + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": { + "maturity": "production", + "tags": [ + "core expectation", + "column aggregate expectation", + "distributional expectation" + ], + "contributors": [ + "@great_expectations" + ], + "requirements": [], + "has_full_test_suite": true, + "manually_reviewed_code": true + } + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": "Expect the p-value of a one-sample Kolmogorov-Smirnov test comparing the column to a named, parameterized theoretical distribution to be greater than or equal to a threshold." + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": [ + "Pandas" + ] + } + } + } + }, + "required": [ + "column", + "distribution" + ], + "additionalProperties": false, + "definitions": { + "ResultFormat": { + "title": "ResultFormat", + "description": "An enumeration.", + "enum": [ + "BOOLEAN_ONLY", + "BASIC", + "COMPLETE", + "SUMMARY" + ], + "type": "string" + }, + "FailureSeverity": { + "title": "FailureSeverity", + "description": "Severity levels for Expectation failures.", + "enum": [ + "critical", + "warning", + "info" + ], + "type": "string" + }, + "Offset": { + "title": "Offset", + "description": "A threshold in which a metric will be considered passable", + "type": "object", + "properties": { + "positive": { + "title": "Positive", + "type": "number" + }, + "negative": { + "title": "Negative", + "type": "number" + } + }, + "required": [ + "positive", + "negative" + ], + "additionalProperties": false + }, + "Window": { + "title": "Window", + "description": "A definition for a temporal window across <`range`> number of previous invocations", + "type": "object", + "properties": { + "constraint_fn": { + "title": "Constraint Fn", + "type": "string" + }, + "parameter_name": { + "title": "Parameter Name", + "type": "string" + }, + "range": { + "title": "Range", + "type": "integer" + }, + "offset": { + "$ref": "#/definitions/Offset" + }, + "strict": { + "title": "Strict", + "default": false, + "type": "boolean" + } + }, + "required": [ + "constraint_fn", + "parameter_name", + "range", + "offset" + ], + "additionalProperties": false + }, + "Column": { + "title": "Column", + "description": "--Public API--\nSpecify the column in a condition statement.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "Operator": { + "title": "Operator", + "description": "An enumeration.", + "enum": [ + "==", + "!=", + "<", + "<=", + ">", + ">=", + "IN", + "NOT_IN" + ], + "type": "string" + }, + "ComparisonCondition": { + "title": "ComparisonCondition", + "description": "--Public API--Condition representing the comparison of a column with a parameter.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "comparison", + "enum": [ + "comparison" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "operator": { + "$ref": "#/definitions/Operator" + }, + "parameter": { + "title": "Parameter" + } + }, + "required": [ + "column", + "operator", + "parameter" + ] + }, + "NullityCondition": { + "title": "NullityCondition", + "description": "--Public API--Condition representing the whether or not a column is null.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "nullity", + "enum": [ + "nullity" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "is_null": { + "title": "Is Null", + "type": "boolean" + } + }, + "required": [ + "column", + "is_null" + ] + }, + "Condition": { + "title": "Condition", + "description": "Base class for conditions.", + "type": "object", + "properties": {} + }, + "AndCondition": { + "title": "AndCondition", + "description": "--Public API--Represents an AND condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "and", + "enum": [ + "and" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "OrCondition": { + "title": "OrCondition", + "description": "--Public API--Represents an OR condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "or", + "enum": [ + "or" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "PassThroughCondition": { + "title": "PassThroughCondition", + "description": "Condition that passes a filter string directly to the execution engine.\n\nThis is used for legacy pandas/spark condition_parser syntax where the\nrow_condition string is passed directly to DataFrame.query() or DataFrame.filter().", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "pass_through", + "enum": [ + "pass_through" + ], + "type": "string" + }, + "pass_through_filter": { + "title": "Pass Through Filter", + "type": "string" + } + }, + "required": [ + "pass_through_filter" + ] + } + } +} diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/__init__.py b/great_expectations/expectations/metrics/column_aggregate_metrics/__init__.py index 6cfc1032ba38..540a4ab580cd 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/__init__.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/__init__.py @@ -1,3 +1,4 @@ +from .column_bootstrapped_ks_test_p_value import ColumnBootstrappedKSTestPValue from .column_descriptive_stats import ColumnDescriptiveStats from .column_distinct_values import ( ColumnDistinctValues, diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_bootstrapped_ks_test_p_value.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_bootstrapped_ks_test_p_value.py index ad4626af6d34..dcbbcc7f9c2f 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_bootstrapped_ks_test_p_value.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_bootstrapped_ks_test_p_value.py @@ -25,7 +25,7 @@ class ColumnBootstrappedKSTestPValue(ColumnAggregateMetricProvider): """MetricProvider Class for Aggregate Standard Deviation metric""" metric_name = "column.bootstrapped_ks_test_p_value" - value_keys = ("partition_object", "p", "bootstrap_sample", "bootstrap_sample_size") + value_keys = ("partition_object", "p", "bootstrap_samples", "bootstrap_sample_size") @column_aggregate_value(engine=PandasExecutionEngine) def _pandas( # noqa: C901 # FIXME CoP diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_parameterized_distribution_ks_test_p_value.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_parameterized_distribution_ks_test_p_value.py index 0c5bae917d8b..5ff06b1b3fef 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_parameterized_distribution_ks_test_p_value.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_parameterized_distribution_ks_test_p_value.py @@ -38,7 +38,10 @@ def _pandas(cls, column, distribution, p_value=0.05, params=None, **kwargs): else: positional_parameters = params - # K-S Test - ks_result = stats.kstest(column, distribution, args=positional_parameters) + # K-S Test. Build a frozen distribution from the requested parameters and test against + # its CDF. Passing a distribution name with `args=` is no longer supported by scipy for + # parameterless CDFs (e.g. ``norm``), so freeze the distribution explicitly instead. + frozen_distribution = getattr(stats, distribution)(*(positional_parameters or ())) + ks_result = stats.kstest(column, frozen_distribution.cdf) return ks_result diff --git a/tasks.py b/tasks.py index 8f769ebc32af..3c868caadc36 100644 --- a/tasks.py +++ b/tasks.py @@ -613,6 +613,10 @@ def type_schema( # noqa: C901 - too complex core.ExpectColumnStdevToBeBetween, core.ExpectColumnSumToBeBetween, core.ExpectColumnKLDivergenceToBeLessThan, + core.ExpectColumnChisquareTestPValueToBeGreaterThan, + core.ExpectColumnBootstrappedKsTestPValueToBeGreaterThan, + core.ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan, + core.ExpectColumnPairCramersPhiValueToBeLessThan, core.ExpectColumnQuantileValuesToBeBetween, core.ExpectColumnValueLengthsToBeBetween, core.ExpectColumnValueLengthsToEqual, diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py new file mode 100644 index 000000000000..258a29ec6034 --- /dev/null +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py @@ -0,0 +1,71 @@ +import pandas as pd +import pytest + +import great_expectations.expectations as gxe +from great_expectations.core.result_format import ResultFormat +from great_expectations.datasource.fluent.interfaces import Batch +from tests.integration.conftest import parameterize_batch_for_data_sources +from tests.integration.data_sources_and_expectations.test_canonical_expectations import ( + JUST_PANDAS_DATA_SOURCES, +) + +# This Expectation relies on a Pandas-only metric (scipy.stats.kstest + numpy bootstrapping), so it +# is exercised against the Pandas data source only. Because it draws random bootstrap samples, the +# observed_value is not deterministic; tests assert on success rather than the exact statistic, and +# use a large bootstrap_sample_size so the pass/fail outcome is stable. + +COL_NAME = "my_col" + +DATA = pd.DataFrame({COL_NAME: [i / 100 for i in range(100)]}) # evenly spread over [0, 1) + +MATCHING_PARTITION = {"bins": [0.0, 0.25, 0.5, 0.75, 1.0], "weights": [0.25, 0.25, 0.25, 0.25]} +SKEWED_PARTITION = {"bins": [0.0, 0.25, 0.5, 0.75, 1.0], "weights": [0.85, 0.05, 0.05, 0.05]} + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_success(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnBootstrappedKsTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object=MATCHING_PARTITION, + p=0.05, + bootstrap_sample_size=100, + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert result.success + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_failure(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnBootstrappedKsTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object=SKEWED_PARTITION, + p=0.05, + bootstrap_sample_size=100, + ) + result = batch_for_datasource.validate(expectation) + assert not result.success + + +@pytest.mark.parametrize( + "suite_param_value,expected_result", + [ + pytest.param(MATCHING_PARTITION, True, id="success"), + pytest.param(SKEWED_PARTITION, False, id="failure"), + ], +) +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_success_with_suite_param_partition_object_( + batch_for_datasource: Batch, suite_param_value: dict, expected_result: bool +) -> None: + suite_param_key = "expect_column_bootstrapped_ks_test_p_value_to_be_greater_than" + expectation = gxe.ExpectColumnBootstrappedKsTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object={"$PARAMETER": suite_param_key}, + p=0.05, + bootstrap_sample_size=100, + result_format=ResultFormat.SUMMARY, + ) + result = batch_for_datasource.validate( + expectation, expectation_parameters={suite_param_key: suite_param_value} + ) + assert result.success == expected_result diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py new file mode 100644 index 000000000000..1d42672e48ce --- /dev/null +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py @@ -0,0 +1,70 @@ +import pandas as pd +import pytest + +import great_expectations.expectations as gxe +from great_expectations.core.result_format import ResultFormat +from great_expectations.datasource.fluent.interfaces import Batch +from tests.integration.conftest import parameterize_batch_for_data_sources +from tests.integration.data_sources_and_expectations.test_canonical_expectations import ( + JUST_PANDAS_DATA_SOURCES, +) + +# This Expectation computes a Chi-square goodness-of-fit p-value with scipy on the column's value +# counts, so it is exercised against the Pandas data source only. + +COL_NAME = "my_col" + +DATA = pd.DataFrame({COL_NAME: ["A"] * 5 + ["B"] * 3 + ["C"] * 2}) + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_success(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnChisquareTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object={"values": ["A", "B", "C"], "weights": [0.5, 0.3, 0.2]}, + p=0.05, + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert result.success + assert result.to_json_dict()["result"] == { + "observed_value": 1.0, + "details": { + "observed_partition": {"values": ["A", "B", "C"], "weights": [0.5, 0.3, 0.2]}, + "expected_partition": {"values": ["A", "B", "C"], "weights": [0.5, 0.3, 0.2]}, + }, + } + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_failure(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnChisquareTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object={"values": ["A", "B", "C"], "weights": [0.1, 0.1, 0.8]}, + p=0.05, + ) + result = batch_for_datasource.validate(expectation) + assert not result.success + + +@pytest.mark.parametrize( + "suite_param_value,expected_result", + [ + pytest.param({"values": ["A", "B", "C"], "weights": [0.5, 0.3, 0.2]}, True, id="success"), + pytest.param({"values": ["A", "B", "C"], "weights": [0.1, 0.1, 0.8]}, False, id="failure"), + ], +) +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_success_with_suite_param_partition_object_( + batch_for_datasource: Batch, suite_param_value: dict, expected_result: bool +) -> None: + suite_param_key = "expect_column_chisquare_test_p_value_to_be_greater_than" + expectation = gxe.ExpectColumnChisquareTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object={"$PARAMETER": suite_param_key}, + p=0.05, + result_format=ResultFormat.SUMMARY, + ) + result = batch_for_datasource.validate( + expectation, expectation_parameters={suite_param_key: suite_param_value} + ) + assert result.success == expected_result diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py new file mode 100644 index 000000000000..79fefa553cd3 --- /dev/null +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py @@ -0,0 +1,82 @@ +import pandas as pd +import pytest + +import great_expectations.expectations as gxe +from great_expectations.core.result_format import ResultFormat +from great_expectations.datasource.fluent.interfaces import Batch +from tests.integration.conftest import parameterize_batch_for_data_sources +from tests.integration.data_sources_and_expectations.test_canonical_expectations import ( + JUST_PANDAS_DATA_SOURCES, +) + +# This Expectation builds a contingency table with pandas and computes Cramér's phi with scipy, so +# it is exercised against the Pandas data source only. + +COL_A = "col_a" +COL_B = "col_b" + +# Independent columns: every combination of (A/B) x (X/Y) appears equally often -> phi == 0. +INDEPENDENT_DATA = pd.DataFrame( + { + COL_A: ["A", "A", "B", "B"] * 25, + COL_B: ["X", "Y", "X", "Y"] * 25, + } +) + +# Perfectly associated columns: col_b is fully determined by col_a -> phi near 1. +ASSOCIATED_DATA = pd.DataFrame( + { + COL_A: ["A"] * 50 + ["B"] * 50, + COL_B: ["X"] * 50 + ["Y"] * 50, + } +) + + +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=INDEPENDENT_DATA +) +def test_success(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, column_B=COL_B, threshold=0.1 + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert result.success + assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(0.0) + + +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=ASSOCIATED_DATA +) +def test_failure(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, column_B=COL_B, threshold=0.1 + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert not result.success + assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(0.98) + + +@pytest.mark.parametrize( + "suite_param_value,expected_result", + [ + pytest.param(0.999, True, id="success"), + pytest.param(0.1, False, id="failure"), + ], +) +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=ASSOCIATED_DATA +) +def test_success_with_suite_param_threshold_( + batch_for_datasource: Batch, suite_param_value: float, expected_result: bool +) -> None: + suite_param_key = "expect_column_pair_cramers_phi_value_to_be_less_than" + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, + column_B=COL_B, + threshold={"$PARAMETER": suite_param_key}, + result_format=ResultFormat.SUMMARY, + ) + result = batch_for_datasource.validate( + expectation, expectation_parameters={suite_param_key: suite_param_value} + ) + assert result.success == expected_result diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py new file mode 100644 index 000000000000..be1faf463ddf --- /dev/null +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py @@ -0,0 +1,67 @@ +import pandas as pd +import pytest + +import great_expectations.expectations as gxe +from great_expectations.core.result_format import ResultFormat +from great_expectations.datasource.fluent.interfaces import Batch +from tests.integration.conftest import parameterize_batch_for_data_sources +from tests.integration.data_sources_and_expectations.test_canonical_expectations import ( + JUST_PANDAS_DATA_SOURCES, +) + +# This Expectation relies on a Pandas-only metric (scipy.stats.kstest), so it is exercised against +# the Pandas data source only. + +COL_NAME = "my_col" + +DATA = pd.DataFrame({COL_NAME: [round(0.1 * i, 1) for i in range(1, 10)]}) + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_success(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan( + column=COL_NAME, + distribution="uniform", + params=[0, 1], + p_value=0.05, + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert result.success + assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(0.9998742840646804) + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_failure(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan( + column=COL_NAME, + distribution="norm", + params={"mean": 5, "std_dev": 1}, + p_value=0.05, + ) + result = batch_for_datasource.validate(expectation) + assert not result.success + + +@pytest.mark.parametrize( + "suite_param_value,expected_result", + [ + pytest.param(0.05, True, id="success"), + pytest.param(0.9999, False, id="failure"), + ], +) +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_success_with_suite_param_p_value_( + batch_for_datasource: Batch, suite_param_value: float, expected_result: bool +) -> None: + suite_param_key = "expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than" + expectation = gxe.ExpectColumnParameterizedDistributionKsTestPValueToBeGreaterThan( + column=COL_NAME, + distribution="uniform", + params=[0, 1], + p_value={"$PARAMETER": suite_param_key}, + result_format=ResultFormat.SUMMARY, + ) + result = batch_for_datasource.validate( + expectation, expectation_parameters={suite_param_key: suite_param_value} + ) + assert result.success == expected_result From 51a5958e95863e9524053a28cdf7603c5f3e2dd7 Mon Sep 17 00:00:00 2001 From: Josh Stauffer <66793731+joshua-stauffer@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:26:07 +0200 Subject: [PATCH 2/5] Cap bootstrap_samples in bootstrapped-KS tests to stay under the unit-test timeout The default 1000 bootstrap rounds pushed each validation to ~1.9s, exceeding the 2s per-test timeout in CI. Use 100 rounds, which keeps each test well under the limit while the matching/skewed partitions still give a stable pass/fail. --- ...olumn_bootstrapped_ks_test_p_value_to_be_greater_than.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py index 258a29ec6034..e511ad087a91 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py @@ -12,7 +12,8 @@ # This Expectation relies on a Pandas-only metric (scipy.stats.kstest + numpy bootstrapping), so it # is exercised against the Pandas data source only. Because it draws random bootstrap samples, the # observed_value is not deterministic; tests assert on success rather than the exact statistic, and -# use a large bootstrap_sample_size so the pass/fail outcome is stable. +# use a large bootstrap_sample_size so the pass/fail outcome is stable. bootstrap_samples is kept +# small so each validation stays well under the unit-test timeout while remaining stable. COL_NAME = "my_col" @@ -28,6 +29,7 @@ def test_success(batch_for_datasource: Batch) -> None: column=COL_NAME, partition_object=MATCHING_PARTITION, p=0.05, + bootstrap_samples=100, bootstrap_sample_size=100, ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) @@ -40,6 +42,7 @@ def test_failure(batch_for_datasource: Batch) -> None: column=COL_NAME, partition_object=SKEWED_PARTITION, p=0.05, + bootstrap_samples=100, bootstrap_sample_size=100, ) result = batch_for_datasource.validate(expectation) @@ -62,6 +65,7 @@ def test_success_with_suite_param_partition_object_( column=COL_NAME, partition_object={"$PARAMETER": suite_param_key}, p=0.05, + bootstrap_samples=100, bootstrap_sample_size=100, result_format=ResultFormat.SUMMARY, ) From c38bb227256a39eed72e2c1254edc778847a36ca Mon Sep 17 00:00:00 2001 From: Josh Stauffer <66793731+joshua-stauffer@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:32:17 +0200 Subject: [PATCH 3/5] Assert on whole result dict in stat-expectation tests for type safety Double-indexing to_json_dict()["result"]["observed_value"] is not type-safe (the value is a union that includes list). Compare the whole result dict with pytest.approx inside, matching the pattern used by the other expectation tests. --- ..._pair_cramers_phi_value_to_be_less_than.py | 26 +++++++++++++++++-- ...tion_ks_test_p_value_to_be_greater_than.py | 10 ++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py index 79fefa553cd3..670256437a3a 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py @@ -41,7 +41,18 @@ def test_success(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert result.success - assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(0.0) + assert result.to_json_dict()["result"] == { + "observed_value": pytest.approx(0.0), + "details": { + "crosstab": { + "row_variable": "col_a", + "column_variable": "col_b", + "rows": ["A", "B"], + "columns": ["X", "Y"], + "counts": [[25, 25], [25, 25]], + } + }, + } @parameterize_batch_for_data_sources( @@ -53,7 +64,18 @@ def test_failure(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert not result.success - assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(0.98) + assert result.to_json_dict()["result"] == { + "observed_value": pytest.approx(0.98), + "details": { + "crosstab": { + "row_variable": "col_a", + "column_variable": "col_b", + "rows": ["A", "B"], + "columns": ["X", "Y"], + "counts": [[50, 0], [0, 50]], + } + }, + } @pytest.mark.parametrize( diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py index be1faf463ddf..5afe93e9d184 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py @@ -27,7 +27,15 @@ def test_success(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert result.success - assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(0.9998742840646804) + assert result.to_json_dict()["result"] == { + "observed_value": pytest.approx(0.9998742840646804), + "details": { + "observed_ks_result": { + "statistic": pytest.approx(0.1), + "pvalue": pytest.approx(0.9998742840646804), + } + }, + } @parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) From d0a19c9fe3b97ca75ae409f098375bf29d788ecf Mon Sep 17 00:00:00 2001 From: Josh Stauffer <66793731+joshua-stauffer@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:46:43 +0200 Subject: [PATCH 4/5] =?UTF-8?q?Harden=20Cram=C3=A9r's=20phi=20binning=20an?= =?UTF-8?q?d=20enrich=20stat-expectation=20result=20details?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix inverted precision in numeric interval binning: small bin widths (< ~0.01) rounded to duplicate interval labels and crashed Categorical.from_codes. Precision now grows as bin width shrinks. - Detect all numeric dtypes (int32/float32, nullable Int64/Float64) via is_numeric_dtype instead of a hardcoded list that silently routed them to the categorical branch; keep booleans on the categorical branch. - Introduce the "(missing)"/"(other)" sentinels without raising on already-categorical columns (cast to object first). - Forward the bootstrapped-KS metric details verbatim so observed/expected partitions and CDFs reach the result. - Document the inclusive boundary comparisons (<= / >=) that mirror legacy behavior for Cramér's phi and parameterized-distribution KS. - Fix the self-contradictory Cramér's phi failing docstring example to use perfectly-associated data. - Add integration tests: numeric binning (incl. small-magnitude), explicit bins/n_bins, nulls, category dtype, and chi-square tail_weight_holdout. --- ...pped_ks_test_p_value_to_be_greater_than.py | 8 +- ..._pair_cramers_phi_value_to_be_less_than.py | 33 +++++- ...tion_ks_test_p_value_to_be_greater_than.py | 3 + ...ColumnPairCramersPhiValueToBeLessThan.json | 2 +- ...pped_ks_test_p_value_to_be_greater_than.py | 11 ++ ...isquare_test_p_value_to_be_greater_than.py | 17 +++ ..._pair_cramers_phi_value_to_be_less_than.py | 100 ++++++++++++++++++ 7 files changed, 164 insertions(+), 10 deletions(-) diff --git a/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py b/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py index f696e02f38b3..f46880c87d6d 100644 --- a/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py +++ b/great_expectations/expectations/core/expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py @@ -345,13 +345,13 @@ def _validate( observed_value = float(metric_result["observed_value"]) details = metric_result.get("details", {}) + # Forward the metric's details verbatim: alongside the bootstrap settings, they carry the + # observed/expected partitions and CDFs, which are what a user needs to see why the test + # failed (and which the legacy V2 result and the sibling chi-square expectation surface). return { "success": bool(observed_value > p), "result": { "observed_value": observed_value, - "details": { - "bootstrap_samples": details.get("bootstrap_samples"), - "bootstrap_sample_size": details.get("bootstrap_sample_size"), - }, + "details": details, }, } diff --git a/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py b/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py index 7de0df155a15..61e3318f6b48 100644 --- a/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py +++ b/great_expectations/expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than.py @@ -77,7 +77,11 @@ def _get_binned_values( # noqa: C901, PLR0912 # FIXME CoP if n_bins is None: n_bins = 10 - if series.dtype in ["int", "float", "int64", "float64"]: + # Bin any real-valued numeric column into intervals. Use is_numeric_dtype so that width-specific + # dtypes (int32/float32) and pandas nullable dtypes (Int64/Float64) are treated as numeric too; + # a hardcoded dtype list silently routed those to the categorical branch. Booleans are excluded + # so they stay categorical (a two-value column is more meaningful ungrouped than binned). + if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series): if bins is not None: sorted_bins = sorted(np.unique(bins)) if np.min(series) < sorted_bins[0]: @@ -93,8 +97,12 @@ def _get_binned_values( # noqa: C901, PLR0912 # FIXME CoP # Make sure max of series is included in rightmost bin edges[-1] = np.nextafter(edges[-1], edges[-1] + 1) - # Create labels for the returned series - precision = int(np.log10(min(edges[1:] - edges[:-1]))) + 2 + # Create labels for the returned series. Round each edge to enough decimal places that + # narrow bins stay distinguishable: the smaller the bin width, the more decimals we need. + # (A naive int(log10(width)) + 2 inverts this and collapses sub-0.01 bins to identical + # labels, which makes Categorical.from_codes raise on duplicate categories.) + min_width = float(min(edges[1:] - edges[:-1])) + precision = max(2, 2 - int(np.floor(np.log10(min_width)))) labels = [ f"[{round(lower, precision)}, {round(upper, precision)})" for lower, upper in itertools.pairwise(edges) @@ -110,10 +118,15 @@ def _get_binned_values( # noqa: C901, PLR0912 # FIXME CoP ) else: + # Cast to object first: fillna/replace below introduce the "(missing)" and "(other)" + # sentinels, and adding a value that isn't an existing category to a categorical-dtype + # series in place raises. Casting to object (a no-op for string/object columns) lets the + # sentinels through; we re-cast to category at the end. + series = series.astype(object) if bins is None: value_counts = series.value_counts(sort=True) if len(value_counts) < n_bins + 1: - return series.fillna("(missing)") + return series.fillna("(missing)").astype("category") else: other_values = sorted(value_counts.index[n_bins:]) replace = dict.fromkeys(other_values, "(other)") @@ -231,7 +244,13 @@ class ExpectColumnPairCramersPhiValueToBeLessThan(BatchExpectation): }} Failing Case: - Input: + Input (test2 is fully determined by test, so the columns are perfectly associated): + test test2 + 0 "A" "X" + 1 "A" "X" + 2 "B" "Y" + 3 "B" "Y" + ExpectColumnPairCramersPhiValueToBeLessThan( column_A="test", column_B="test2", @@ -434,6 +453,10 @@ def _validate( # See e.g. https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V cramers_phi = float(max(min(np.sqrt(chi2_statistic / n / (min_dimension - 1)), 1), 0)) + # Success is inclusive at the boundary (phi == threshold passes), matching the legacy V2 + # behavior of this expectation and the sibling ExpectColumnKlDivergenceToBeLessThan. This is + # deliberate: phi is clamped to exactly 0.0 for degenerate tables (n == 0 or a constant + # column), and threshold=0 must still succeed in that case. return { "success": bool(cramers_phi <= threshold), "result": { diff --git a/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py b/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py index 70e8927c6a71..cc546979c489 100644 --- a/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py +++ b/great_expectations/expectations/core/expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than.py @@ -332,6 +332,9 @@ def _validate( observed_statistic = float(ks_result[0]) observed_p_value = float(ks_result[1]) + # Success is inclusive at the boundary (p-value == p_value passes), matching the legacy V2 + # behavior of this expectation. (The sibling chi-square and bootstrapped-KS expectations use + # a strict comparison, each preserving its own legacy behavior.) return { "success": bool(observed_p_value >= p_value), "result": { diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json index f3c2e5f17e79..23f4553bfb35 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairCramersPhiValueToBeLessThan.json @@ -1,6 +1,6 @@ { "title": "Expect column pair Cram\u00e9r's phi value to be less than", - "description": "Expect the Cram\u00e9r's phi (V) measure of association between two columns to be less than or equal to a threshold.\n\nExpectColumnPairCramersPhiValueToBeLessThan is a Batch Expectation.\n\nBatchExpectations are one of the most common types of Expectation.\nThey are evaluated for an entire Batch, and answer a semantic question about the Batch itself.\n\nCram\u00e9r's phi (also written Cram\u00e9r's V) measures the strength of association between two categorical variables, derived from the Chi-square statistic of their contingency table. A value near 0 indicates the columns are (close to) independent; larger values indicate stronger association. Numeric columns are discretized into bins before the table is built.\n\nArgs:\n column_A (str): The first column name.\n column_B (str): The second column name.\n threshold (float): The maximum Cram\u00e9r's phi value for which to return success=True. Cram\u00e9r's phi ranges from 0 (no association / independent) to 1 (perfect association). Defaults to 0.1.\n bins_A (list or None): Explicit bin edges (numeric column) or groups of values (categorical column) used to discretize column_A before building the contingency table.\n bins_B (list or None): As bins_A, but for column_B.\n n_bins_A (int or None): The number of bins to use for column_A when bins_A is not provided. Defaults to 10.\n n_bins_B (int or None): As n_bins_A, but for column_B.\n\nOther Parameters:\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nNotes:\n * observed_value field in the result object is customized for this expectation to be the Cram\u00e9r's phi (V) value.\n * details.crosstab is customized for this expectation to be a serializable representation of the contingency table between column_A and column_B.\n * The Expectation succeeds when the observed Cram\u00e9r's phi is less than or equal to threshold.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n test test2\n 0 \"A\" \"X\"\n 1 \"A\" \"Y\"\n 2 \"B\" \"X\"\n 3 \"B\" \"Y\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnPairCramersPhiValueToBeLessThan(\n column_A=\"test\",\n column_B=\"test2\",\n threshold=0.1\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnPairCramersPhiValueToBeLessThan(\n column_A=\"test\",\n column_B=\"test2\",\n threshold=0.1\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 1.0\n },\n \"meta\": {},\n \"success\": false\n }", + "description": "Expect the Cram\u00e9r's phi (V) measure of association between two columns to be less than or equal to a threshold.\n\nExpectColumnPairCramersPhiValueToBeLessThan is a Batch Expectation.\n\nBatchExpectations are one of the most common types of Expectation.\nThey are evaluated for an entire Batch, and answer a semantic question about the Batch itself.\n\nCram\u00e9r's phi (also written Cram\u00e9r's V) measures the strength of association between two categorical variables, derived from the Chi-square statistic of their contingency table. A value near 0 indicates the columns are (close to) independent; larger values indicate stronger association. Numeric columns are discretized into bins before the table is built.\n\nArgs:\n column_A (str): The first column name.\n column_B (str): The second column name.\n threshold (float): The maximum Cram\u00e9r's phi value for which to return success=True. Cram\u00e9r's phi ranges from 0 (no association / independent) to 1 (perfect association). Defaults to 0.1.\n bins_A (list or None): Explicit bin edges (numeric column) or groups of values (categorical column) used to discretize column_A before building the contingency table.\n bins_B (list or None): As bins_A, but for column_B.\n n_bins_A (int or None): The number of bins to use for column_A when bins_A is not provided. Defaults to 10.\n n_bins_B (int or None): As n_bins_A, but for column_B.\n\nOther Parameters:\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nNotes:\n * observed_value field in the result object is customized for this expectation to be the Cram\u00e9r's phi (V) value.\n * details.crosstab is customized for this expectation to be a serializable representation of the contingency table between column_A and column_B.\n * The Expectation succeeds when the observed Cram\u00e9r's phi is less than or equal to threshold.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n test test2\n 0 \"A\" \"X\"\n 1 \"A\" \"Y\"\n 2 \"B\" \"X\"\n 3 \"B\" \"Y\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnPairCramersPhiValueToBeLessThan(\n column_A=\"test\",\n column_B=\"test2\",\n threshold=0.1\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 0.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input (test2 is fully determined by test, so the columns are perfectly associated):\n test test2\n 0 \"A\" \"X\"\n 1 \"A\" \"X\"\n 2 \"B\" \"Y\"\n 3 \"B\" \"Y\"\n\n ExpectColumnPairCramersPhiValueToBeLessThan(\n column_A=\"test\",\n column_B=\"test2\",\n threshold=0.1\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"observed_value\": 1.0\n },\n \"meta\": {},\n \"success\": false\n }", "type": "object", "properties": { "id": { diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py index e511ad087a91..4229165f562c 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py @@ -34,6 +34,17 @@ def test_success(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert result.success + # The full metric details are surfaced so a user can inspect why the test passed or failed: + # bootstrap settings plus the observed/expected partitions and CDFs. + details = result.to_json_dict()["result"]["details"] + assert set(details) >= { + "bootstrap_samples", + "bootstrap_sample_size", + "observed_partition", + "expected_partition", + "observed_cdf", + "expected_cdf", + } @parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py index 1d42672e48ce..86e6092c8ea5 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py @@ -68,3 +68,20 @@ def test_success_with_suite_param_partition_object_( expectation, expectation_parameters={suite_param_key: suite_param_value} ) assert result.success == expected_result + + +@parameterize_batch_for_data_sources(data_source_configs=JUST_PANDAS_DATA_SOURCES, data=DATA) +def test_tail_weight_holdout_covers_unexpected_values(batch_for_datasource: Batch) -> None: + # The partition only lists A and B, but the data also contains C. tail_weight_holdout reserves + # probability mass for such unexpected values so the test does not treat C as impossible. + expectation = gxe.ExpectColumnChisquareTestPValueToBeGreaterThan( + column=COL_NAME, + partition_object={"values": ["A", "B"], "weights": [0.6, 0.4]}, + p=0.05, + tail_weight_holdout=0.25, + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert result.success + details = result.to_json_dict()["result"]["details"] + # C is folded into the expected partition via the holdout rather than being dropped. + assert "C" in details["expected_partition"]["values"] diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py index 670256437a3a..644ecf365630 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py @@ -1,3 +1,4 @@ +import numpy as np import pandas as pd import pytest @@ -31,6 +32,41 @@ } ) +# Small-magnitude numeric columns (bin width ~0.001) that are perfectly associated. This exercises +# the numeric interval-binning branch and, because the bin widths are far below 0.01, guards against +# the interval-label rounding collapsing adjacent edges into duplicate categories. +SMALL_MAGNITUDE_NUMERIC_DATA = pd.DataFrame( + { + COL_A: np.linspace(0, 0.01, 100), + COL_B: np.linspace(0, 0.01, 100), + } +) + +# Wider numeric columns for exercising explicit bin edges (bins_A) and a bin count (n_bins_B). +NUMERIC_ASSOCIATED_DATA = pd.DataFrame( + { + COL_A: np.linspace(0, 10, 100), + COL_B: np.linspace(0, 10, 100), + } +) + +# Numeric columns containing nulls, to exercise the "(missing)" bucket in the numeric branch. +NUMERIC_WITH_NULLS_DATA = pd.DataFrame( + { + COL_A: [1.0, 2.0, 3.0, None, 5.0] * 10, + COL_B: [1.0, 2.0, 3.0, 4.0, None] * 10, + } +) + +# Columns that are already ``category`` dtype and contain nulls, to exercise the categorical branch: +# introducing the "(missing)" sentinel must not raise on a categorical-dtype series. +CATEGORY_WITH_NULLS_DATA = pd.DataFrame( + { + COL_A: pd.Series(["A"] * 40 + ["B"] * 40 + [None] * 20, dtype="category"), + COL_B: pd.Series(["X"] * 40 + ["Y"] * 40 + [None] * 20, dtype="category"), + } +) + @parameterize_batch_for_data_sources( data_source_configs=JUST_PANDAS_DATA_SOURCES, data=INDEPENDENT_DATA @@ -102,3 +138,67 @@ def test_success_with_suite_param_threshold_( expectation, expectation_parameters={suite_param_key: suite_param_value} ) assert result.success == expected_result + + +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=SMALL_MAGNITUDE_NUMERIC_DATA +) +def test_numeric_columns_small_magnitude(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, column_B=COL_B, threshold=0.1 + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + # Perfectly associated numeric columns -> phi ~ 1.0, well above the threshold. + assert not result.success + result_dict = result.to_json_dict()["result"] + assert result_dict["observed_value"] == pytest.approx(1.0) + # Default n_bins=10 -> a 10x10 contingency table; the small bin width must not collapse labels. + crosstab = result_dict["details"]["crosstab"] + assert len(crosstab["rows"]) == 10 + assert len(crosstab["columns"]) == 10 + + +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=NUMERIC_ASSOCIATED_DATA +) +def test_numeric_columns_with_explicit_bins_and_n_bins(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, + column_B=COL_B, + threshold=0.1, + bins_A=[2.5, 5.0, 7.5], + n_bins_B=4, + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert not result.success + assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(1.0) + + +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=NUMERIC_WITH_NULLS_DATA +) +def test_numeric_columns_with_nulls(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, column_B=COL_B, threshold=0.1 + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert not result.success + crosstab = result.to_json_dict()["result"]["details"]["crosstab"] + # Nulls are collected into a dedicated "(missing)" bucket on both axes. + assert "(missing)" in crosstab["rows"] + assert "(missing)" in crosstab["columns"] + + +@parameterize_batch_for_data_sources( + data_source_configs=JUST_PANDAS_DATA_SOURCES, data=CATEGORY_WITH_NULLS_DATA +) +def test_category_dtype_columns_with_nulls(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnPairCramersPhiValueToBeLessThan( + column_A=COL_A, column_B=COL_B, threshold=0.1 + ) + result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) + assert not result.success + assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(1.0) + crosstab = result.to_json_dict()["result"]["details"]["crosstab"] + assert "(missing)" in crosstab["rows"] + assert "(missing)" in crosstab["columns"] From 026ced5aff41612674751e90b8f57da5ab2c04a8 Mon Sep 17 00:00:00 2001 From: Josh Stauffer <66793731+joshua-stauffer@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:55:35 +0200 Subject: [PATCH 5/5] Use result.result for typed test assertions to satisfy mypy Chained subscripts into to_json_dict()["result"] return JSONValues (a union) and fail mypy's __getitem__ overload check. Access result.result (a plain dict) behind an is-not-None guard instead, matching the pattern used elsewhere in the integration tests. --- ...rapped_ks_test_p_value_to_be_greater_than.py | 3 ++- ...chisquare_test_p_value_to_be_greater_than.py | 3 ++- ...mn_pair_cramers_phi_value_to_be_less_than.py | 17 ++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py index 4229165f562c..832131923fc6 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_bootstrapped_ks_test_p_value_to_be_greater_than.py @@ -36,7 +36,8 @@ def test_success(batch_for_datasource: Batch) -> None: assert result.success # The full metric details are surfaced so a user can inspect why the test passed or failed: # bootstrap settings plus the observed/expected partitions and CDFs. - details = result.to_json_dict()["result"]["details"] + assert result.result is not None + details = result.result["details"] assert set(details) >= { "bootstrap_samples", "bootstrap_sample_size", diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py index 86e6092c8ea5..b78112e12b30 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_chisquare_test_p_value_to_be_greater_than.py @@ -82,6 +82,7 @@ def test_tail_weight_holdout_covers_unexpected_values(batch_for_datasource: Batc ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert result.success - details = result.to_json_dict()["result"]["details"] + assert result.result is not None + details = result.result["details"] # C is folded into the expected partition via the holdout rather than being dropped. assert "C" in details["expected_partition"]["values"] diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py index 644ecf365630..8915364236c6 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_pair_cramers_phi_value_to_be_less_than.py @@ -150,10 +150,10 @@ def test_numeric_columns_small_magnitude(batch_for_datasource: Batch) -> None: result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) # Perfectly associated numeric columns -> phi ~ 1.0, well above the threshold. assert not result.success - result_dict = result.to_json_dict()["result"] - assert result_dict["observed_value"] == pytest.approx(1.0) + assert result.result is not None + assert result.result["observed_value"] == pytest.approx(1.0) # Default n_bins=10 -> a 10x10 contingency table; the small bin width must not collapse labels. - crosstab = result_dict["details"]["crosstab"] + crosstab = result.result["details"]["crosstab"] assert len(crosstab["rows"]) == 10 assert len(crosstab["columns"]) == 10 @@ -171,7 +171,8 @@ def test_numeric_columns_with_explicit_bins_and_n_bins(batch_for_datasource: Bat ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert not result.success - assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(1.0) + assert result.result is not None + assert result.result["observed_value"] == pytest.approx(1.0) @parameterize_batch_for_data_sources( @@ -183,7 +184,8 @@ def test_numeric_columns_with_nulls(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert not result.success - crosstab = result.to_json_dict()["result"]["details"]["crosstab"] + assert result.result is not None + crosstab = result.result["details"]["crosstab"] # Nulls are collected into a dedicated "(missing)" bucket on both axes. assert "(missing)" in crosstab["rows"] assert "(missing)" in crosstab["columns"] @@ -198,7 +200,8 @@ def test_category_dtype_columns_with_nulls(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation, result_format=ResultFormat.COMPLETE) assert not result.success - assert result.to_json_dict()["result"]["observed_value"] == pytest.approx(1.0) - crosstab = result.to_json_dict()["result"]["details"]["crosstab"] + assert result.result is not None + assert result.result["observed_value"] == pytest.approx(1.0) + crosstab = result.result["details"]["crosstab"] assert "(missing)" in crosstab["rows"] assert "(missing)" in crosstab["columns"]