From 7146690a11aed411877f39db1dfecf3cb54251fa Mon Sep 17 00:00:00 2001 From: nanjeshramesh Date: Tue, 28 Jul 2026 01:44:51 -0700 Subject: [PATCH 1/3] Promote ExpectColumnValuesToMatchStrftimeFormat to supported-core Brings the expectation up to the Gallery-supported bar: fixes the broken docstring (the FAILURE_SEVERITY_DESCRIPTION token was never interpolated), adds Gallery support metadata and a generated JSON schema, and declares Pandas + Spark as the supported backend matrix (SQL is out of scope for now, per the discussion in the issue - strftime tokens don't map cleanly onto SQL dialects' date-format models). Extends the integration tests to cover Pandas alongside the existing Spark cases, plus mostly-threshold coverage. Closes #12004 Co-Authored-By: Claude Sonnet 5 --- ..._column_values_to_match_strftime_format.py | 153 +++++- ...pectColumnValuesToMatchStrftimeFormat.json | 448 ++++++++++++++++++ tasks.py | 1 + ..._column_values_to_match_strftime_format.py | 49 +- 4 files changed, 634 insertions(+), 17 deletions(-) create mode 100644 great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json diff --git a/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py b/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py index 218b8cf89592..023630e50df2 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime, timezone -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -12,6 +12,12 @@ _style_row_condition, render_suite_parameter_string, ) +from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources +from great_expectations.expectations.model_field_descriptions import ( + COLUMN_DESCRIPTION, + FAILURE_SEVERITY_DESCRIPTION, + MOSTLY_DESCRIPTION, +) from great_expectations.render import LegacyRendererType, RenderedStringTemplateContent from great_expectations.render.renderer.renderer import renderer from great_expectations.render.renderer_configuration import ( @@ -33,25 +39,37 @@ ) from great_expectations.render.renderer_configuration import AddParamArgs +EXPECTATION_SHORT_DESCRIPTION = ( + "Expect the column entries to be strings representing a date or time with a given format." +) +STRFTIME_FORMAT_DESCRIPTION = "A strftime format string to use for matching." +DATA_QUALITY_ISSUES = [DataQualityIssues.VALIDITY.value] +SUPPORTED_DATA_SOURCES = [ + SupportedDataSources.PANDAS.value, + SupportedDataSources.SPARK.value, +] + class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): - """Expect the column entries to be strings representing a date or time with a given format. + __doc__ = f"""{EXPECTATION_SHORT_DESCRIPTION} ExpectColumnValuesToMatchStrftimeFormat is a \ Column Map Expectation. + Column Map Expectations are one of the most common types of Expectation. + They are evaluated for a single column and ask a yes/no question for every row in that column. + Based on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. + Args: column (str): \ - The column name. + {COLUMN_DESCRIPTION} strftime_format (str or SuiteParameterDict): \ - A strftime format string to use for matching + {STRFTIME_FORMAT_DESCRIPTION} - Keyword Args: + Other Parameters: mostly (None or a float between 0 and 1): \ - Successful if at least mostly fraction of values match the expectation. \ + {MOSTLY_DESCRIPTION} \ For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). - - Other Parameters: result_format (str or None): \ Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). @@ -69,9 +87,90 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result) Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta. + + Supported Data Sources: + [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[1]}](https://docs.greatexpectations.io/docs/application_integration_support/) + + SQL data sources are not currently supported: strftime format tokens do not map cleanly \ + onto the date-format models of SQL dialects. + + Data Quality Issues: + {DATA_QUALITY_ISSUES[0]} + + Example Data: + event_date + 0 "2024-01-15" + 1 "2024-06-20" + 2 "not-a-date" + + Code Examples: + Passing Case: + Input: + ExpectColumnValuesToMatchStrftimeFormat( + column="event_date", + strftime_format="%Y-%m-%d", + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "element_count": 3, + "unexpected_count": 1, + "unexpected_percent": 33.33333333333333, + "partial_unexpected_list": [ + "not-a-date" + ], + "missing_count": 0, + "missing_percent": 0.0, + "unexpected_percent_total": 33.33333333333333, + "unexpected_percent_nonmissing": 33.33333333333333 + }}, + "meta": {{}}, + "success": false + }} + + Failing Case: + Input: + ExpectColumnValuesToMatchStrftimeFormat( + column="event_date", + strftime_format="%m/%d/%Y", + ) + + Output: + {{ + "exception_info": {{ + "raised_exception": false, + "exception_traceback": null, + "exception_message": null + }}, + "result": {{ + "element_count": 3, + "unexpected_count": 3, + "unexpected_percent": 100.0, + "partial_unexpected_list": [ + "2024-01-15", + "2024-06-20", + "not-a-date" + ], + "missing_count": 0, + "missing_percent": 0.0, + "unexpected_percent_total": 100.0, + "unexpected_percent_nonmissing": 100.0 + }}, + "meta": {{}}, + "success": false + }} """ # noqa: E501 # FIXME CoP - strftime_format: Union[str, SuiteParameterDict] + strftime_format: Union[str, SuiteParameterDict] = pydantic.Field( + description=STRFTIME_FORMAT_DESCRIPTION + ) @pydantic.validator("strftime_format") def validate_strftime_format( @@ -88,7 +187,7 @@ def validate_strftime_format( return strftime_format - library_metadata = { + library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -98,6 +197,7 @@ def validate_strftime_format( "has_full_test_suite": True, "manually_reviewed_code": True, } + _library_metadata = library_metadata map_metric = "column_values.match_strftime_format" success_keys = ( @@ -109,6 +209,39 @@ def validate_strftime_format( "strftime_format", ) + class Config: + title = "Expect column values to match strftime format" + + @staticmethod + def schema_extra( + schema: Dict[str, Any], model: Type[ExpectColumnValuesToMatchStrftimeFormat] + ) -> None: + ColumnMapExpectation.Config.schema_extra(schema, model) + schema["properties"]["metadata"]["properties"].update( + { + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": DATA_QUALITY_ISSUES, + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": model._library_metadata, + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": EXPECTATION_SHORT_DESCRIPTION, + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": SUPPORTED_DATA_SOURCES, + }, + } + ) + @classmethod def _prescriptive_template( cls, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json new file mode 100644 index 000000000000..9c76a8c10753 --- /dev/null +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json @@ -0,0 +1,448 @@ +{ + "title": "Expect column values to match strftime format", + "description": "Expect the column entries to be strings representing a date or time with a given format.\n\nExpectColumnValuesToMatchStrftimeFormat is a Column Map Expectation.\n\nColumn Map Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column and ask a yes/no question for every row in that column.\nBased on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid.\n\nArgs:\n column (str): The column name.\n strftime_format (str or SuiteParameterDict): A strftime format string to use for matching.\n\nOther Parameters:\n mostly (None or a float between 0 and 1): Successful if at least `mostly` fraction of values match the Expectation. For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly).\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Spark](https://docs.greatexpectations.io/docs/application_integration_support/)\n\n SQL data sources are not currently supported: strftime format tokens do not map cleanly onto the date-format models of SQL dialects.\n\nData Quality Issues:\n Validity\n\nExample Data:\n event_date\n 0 \"2024-01-15\"\n 1 \"2024-06-20\"\n 2 \"not-a-date\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 1,\n \"unexpected_percent\": 33.33333333333333,\n \"partial_unexpected_list\": [\n \"not-a-date\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 33.33333333333333,\n \"unexpected_percent_nonmissing\": 33.33333333333333\n },\n \"meta\": {},\n \"success\": false\n }\n\n Failing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%m/%d/%Y\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 3,\n \"unexpected_percent\": 100.0,\n \"partial_unexpected_list\": [\n \"2024-01-15\",\n \"2024-06-20\",\n \"not-a-date\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 100.0,\n \"unexpected_percent_nonmissing\": 100.0\n },\n \"meta\": {},\n \"success\": false\n }", + "type": "object", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "meta": { + "title": "Meta", + "type": "object" + }, + "notes": { + "title": "Notes", + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "result_format": { + "title": "Result Format", + "default": "BASIC", + "anyOf": [ + { + "$ref": "#/definitions/ResultFormat" + }, + { + "type": "object" + } + ] + }, + "description": { + "title": "Description", + "description": "A short description of your Expectation", + "type": "string" + }, + "catch_exceptions": { + "title": "Catch Exceptions", + "default": true, + "type": "boolean" + }, + "rendered_content": { + "title": "Rendered Content", + "type": "array", + "items": { + "type": "object" + } + }, + "severity": { + "description": "Indicate the impact of this Expectation failing. Severity levels can be used to trigger different alerting patterns and actions.", + "default": "critical", + "allOf": [ + { + "$ref": "#/definitions/FailureSeverity" + } + ] + }, + "windows": { + "title": "Windows", + "description": "Definition(s) for evaluation of temporal windows", + "type": "array", + "items": { + "$ref": "#/definitions/Window" + } + }, + "batch_id": { + "title": "Batch Id", + "type": "string" + }, + "column": { + "title": "Column", + "description": "The column name.", + "minLength": 1, + "type": "string" + }, + "mostly": { + "title": "Mostly", + "description": "Successful if at least `mostly` fraction of values match the Expectation.", + "default": 1, + "anyOf": [ + { + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + }, + { + "type": "object" + } + ], + "multipleOf": 0.01 + }, + "row_condition": { + "title": "Row Condition", + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ComparisonCondition" + }, + { + "$ref": "#/definitions/NullityCondition" + }, + { + "$ref": "#/definitions/AndCondition" + }, + { + "$ref": "#/definitions/OrCondition" + }, + { + "$ref": "#/definitions/PassThroughCondition" + } + ] + }, + "condition_parser": { + "title": "Condition Parser", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" + }, + "strftime_format": { + "title": "Strftime Format", + "description": "A strftime format string to use for matching.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] + }, + "metadata": { + "type": "object", + "properties": { + "expectation_class": { + "title": "Expectation Class", + "type": "string", + "const": "ExpectColumnValuesToMatchStrftimeFormat" + }, + "expectation_type": { + "title": "Expectation Type", + "type": "string", + "const": "expect_column_values_to_match_strftime_format" + }, + "domain_type": { + "title": "Domain Type", + "type": "string", + "const": "column", + "description": "Column Map" + }, + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": [ + "Validity" + ] + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": { + "maturity": "production", + "tags": [ + "core expectation", + "column map expectation" + ], + "contributors": [ + "@great_expectations" + ], + "requirements": [], + "has_full_test_suite": true, + "manually_reviewed_code": true + } + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": "Expect the column entries to be strings representing a date or time with a given format." + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": [ + "Pandas", + "Spark" + ] + } + } + } + }, + "required": [ + "column", + "strftime_format" + ], + "additionalProperties": false, + "definitions": { + "ResultFormat": { + "title": "ResultFormat", + "description": "An enumeration.", + "enum": [ + "BOOLEAN_ONLY", + "BASIC", + "COMPLETE", + "SUMMARY" + ], + "type": "string" + }, + "FailureSeverity": { + "title": "FailureSeverity", + "description": "Severity levels for Expectation failures.", + "enum": [ + "critical", + "warning", + "info" + ], + "type": "string" + }, + "Offset": { + "title": "Offset", + "description": "A threshold in which a metric will be considered passable", + "type": "object", + "properties": { + "positive": { + "title": "Positive", + "type": "number" + }, + "negative": { + "title": "Negative", + "type": "number" + } + }, + "required": [ + "positive", + "negative" + ], + "additionalProperties": false + }, + "Window": { + "title": "Window", + "description": "A definition for a temporal window across <`range`> number of previous invocations", + "type": "object", + "properties": { + "constraint_fn": { + "title": "Constraint Fn", + "type": "string" + }, + "parameter_name": { + "title": "Parameter Name", + "type": "string" + }, + "range": { + "title": "Range", + "type": "integer" + }, + "offset": { + "$ref": "#/definitions/Offset" + }, + "strict": { + "title": "Strict", + "default": false, + "type": "boolean" + } + }, + "required": [ + "constraint_fn", + "parameter_name", + "range", + "offset" + ], + "additionalProperties": false + }, + "Column": { + "title": "Column", + "description": "--Public API--\nSpecify the column in a condition statement.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "Operator": { + "title": "Operator", + "description": "An enumeration.", + "enum": [ + "==", + "!=", + "<", + "<=", + ">", + ">=", + "IN", + "NOT_IN" + ], + "type": "string" + }, + "ComparisonCondition": { + "title": "ComparisonCondition", + "description": "--Public API--Condition representing the comparison of a column with a parameter.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "comparison", + "enum": [ + "comparison" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "operator": { + "$ref": "#/definitions/Operator" + }, + "parameter": { + "title": "Parameter" + } + }, + "required": [ + "column", + "operator", + "parameter" + ] + }, + "NullityCondition": { + "title": "NullityCondition", + "description": "--Public API--Condition representing the whether or not a column is null.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "nullity", + "enum": [ + "nullity" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "is_null": { + "title": "Is Null", + "type": "boolean" + } + }, + "required": [ + "column", + "is_null" + ] + }, + "Condition": { + "title": "Condition", + "description": "Base class for conditions.", + "type": "object", + "properties": {} + }, + "AndCondition": { + "title": "AndCondition", + "description": "--Public API--Represents an AND condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "and", + "enum": [ + "and" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "OrCondition": { + "title": "OrCondition", + "description": "--Public API--Represents an OR condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "or", + "enum": [ + "or" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "PassThroughCondition": { + "title": "PassThroughCondition", + "description": "Condition that passes a filter string directly to the execution engine.\n\nThis is used for legacy pandas/spark condition_parser syntax where the\nrow_condition string is passed directly to DataFrame.query() or DataFrame.filter().", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "pass_through", + "enum": [ + "pass_through" + ], + "type": "string" + }, + "pass_through_filter": { + "title": "Pass Through Filter", + "type": "string" + } + }, + "required": [ + "pass_through_filter" + ] + } + } +} diff --git a/tasks.py b/tasks.py index c4535ee81ec6..04980a2a8087 100644 --- a/tasks.py +++ b/tasks.py @@ -622,6 +622,7 @@ def type_schema( # noqa: C901 - too complex core.ExpectColumnValuesToMatchLikePatternList, core.ExpectColumnValuesToMatchRegex, core.ExpectColumnValuesToMatchRegexList, + core.ExpectColumnValuesToMatchStrftimeFormat, core.ExpectColumnValuesToNotBeInSet, core.ExpectColumnValuesToNotBeNull, core.ExpectColumnValuesToNotMatchLikePattern, diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_match_strftime_format.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_match_strftime_format.py index a0b3d456feef..9c2a5f03ed10 100644 --- a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_match_strftime_format.py +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_match_strftime_format.py @@ -1,26 +1,34 @@ from typing import Sequence import pandas as pd -import pytest import great_expectations.expectations as gxe from great_expectations.datasource.fluent.interfaces import Batch from tests.integration.conftest import parameterize_batch_for_data_sources from tests.integration.test_utils.data_source_config import ( + PandasFilesystemCsvDatasourceTestConfig, SparkFilesystemCsvDatasourceTestConfig, ) from tests.integration.test_utils.data_source_config.base import DataSourceTestConfig -pyspark_types = pytest.importorskip("pyspark.sql.types") +TIMESTAMPS = "timestamps" +MIXED_FORMAT_TIMESTAMPS = "mixed_format_timestamps" + +try: + from great_expectations.compatibility.pyspark import types as PYSPARK_TYPES + + SPARK_COLUMN_TYPES = { + TIMESTAMPS: PYSPARK_TYPES.StringType, + MIXED_FORMAT_TIMESTAMPS: PYSPARK_TYPES.StringType, + } +except ModuleNotFoundError: + SPARK_COLUMN_TYPES = {} SUPPORTED_DATA_SOURCES: Sequence[DataSourceTestConfig] = [ - SparkFilesystemCsvDatasourceTestConfig( - column_types={"timestamps": pyspark_types.StringType}, - ), + PandasFilesystemCsvDatasourceTestConfig(), + SparkFilesystemCsvDatasourceTestConfig(column_types=SPARK_COLUMN_TYPES), ] -TIMESTAMPS = "timestamps" - DATA = pd.DataFrame( { TIMESTAMPS: [ @@ -28,6 +36,11 @@ "2026-06-20T14:45:00+0000", "2026-12-31T23:59:59+0000", ], + MIXED_FORMAT_TIMESTAMPS: [ + "2026-01-15T10:30:00+0000", + "2026-06-20T14:45:00+0000", + "not-a-timestamp", + ], } ) @@ -56,3 +69,25 @@ def test_non_matching_format_failure(batch_for_datasource: Batch) -> None: ) result = batch_for_datasource.validate(expectation) assert not result.success + + +@parameterize_batch_for_data_sources(data_source_configs=SUPPORTED_DATA_SOURCES, data=DATA) +def test_mostly_threshold_met_success(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnValuesToMatchStrftimeFormat( + column=MIXED_FORMAT_TIMESTAMPS, + strftime_format="%Y-%m-%dT%H:%M:%S%z", + mostly=0.5, + ) + result = batch_for_datasource.validate(expectation) + assert result.success + + +@parameterize_batch_for_data_sources(data_source_configs=SUPPORTED_DATA_SOURCES, data=DATA) +def test_mostly_threshold_not_met_failure(batch_for_datasource: Batch) -> None: + expectation = gxe.ExpectColumnValuesToMatchStrftimeFormat( + column=MIXED_FORMAT_TIMESTAMPS, + strftime_format="%Y-%m-%dT%H:%M:%S%z", + mostly=0.9, + ) + result = batch_for_datasource.validate(expectation) + assert not result.success From ed298be8b1fe4bc33bff0d0f079e0caff86a76b5 Mon Sep 17 00:00:00 2001 From: nanjeshramesh Date: Tue, 4 Aug 2026 14:24:40 -0700 Subject: [PATCH 2/3] Address review: fix docstring examples and formatting - Rework the Gallery Code Examples to use two columns (event_date / invalid_date), so the Passing Case genuinely passes (success: true) and the Failing Case genuinely fails, following the pattern in ExpectColumnValuesToMatchRegex. Both examples were run against the actual expectation to confirm the JSON output is accurate. - Move the SQL-unsupported caveat out of the indented Supported Data Sources block (where its line continuation rendered with extra whitespace) and onto the end of the introductory Column Map Expectations paragraph instead. - Append "Default 1." to the mostly parameter description, matching the regex exemplar. Regenerated the JSON schema to match the updated docstring. Co-Authored-By: Claude Sonnet 5 --- ..._column_values_to_match_strftime_format.py | 41 +++++++++---------- ...pectColumnValuesToMatchStrftimeFormat.json | 2 +- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py b/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py index 023630e50df2..acd84a946c91 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py @@ -58,7 +58,9 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): Column Map Expectations are one of the most common types of Expectation. They are evaluated for a single column and ask a yes/no question for every row in that column. - Based on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. + Based on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. \ + SQL data sources are not currently supported: strftime format tokens do not map cleanly \ + onto the date-format models of SQL dialects. Args: column (str): \ @@ -69,7 +71,7 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): Other Parameters: mostly (None or a float between 0 and 1): \ {MOSTLY_DESCRIPTION} \ - For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). + For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). Default 1. result_format (str or None): \ Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). @@ -92,17 +94,14 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) [{SUPPORTED_DATA_SOURCES[1]}](https://docs.greatexpectations.io/docs/application_integration_support/) - SQL data sources are not currently supported: strftime format tokens do not map cleanly \ - onto the date-format models of SQL dialects. - Data Quality Issues: {DATA_QUALITY_ISSUES[0]} Example Data: - event_date - 0 "2024-01-15" - 1 "2024-06-20" - 2 "not-a-date" + event_date invalid_date + 0 "2024-01-15" "01/15/2024" + 1 "2024-06-20" "06/20/2024" + 2 "2024-12-31" "12/31/2024" Code Examples: Passing Case: @@ -121,25 +120,23 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): }}, "result": {{ "element_count": 3, - "unexpected_count": 1, - "unexpected_percent": 33.33333333333333, - "partial_unexpected_list": [ - "not-a-date" - ], + "unexpected_count": 0, + "unexpected_percent": 0.0, + "partial_unexpected_list": [], "missing_count": 0, "missing_percent": 0.0, - "unexpected_percent_total": 33.33333333333333, - "unexpected_percent_nonmissing": 33.33333333333333 + "unexpected_percent_total": 0.0, + "unexpected_percent_nonmissing": 0.0 }}, "meta": {{}}, - "success": false + "success": true }} Failing Case: Input: ExpectColumnValuesToMatchStrftimeFormat( - column="event_date", - strftime_format="%m/%d/%Y", + column="invalid_date", + strftime_format="%Y-%m-%d", ) Output: @@ -154,9 +151,9 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): "unexpected_count": 3, "unexpected_percent": 100.0, "partial_unexpected_list": [ - "2024-01-15", - "2024-06-20", - "not-a-date" + "01/15/2024", + "06/20/2024", + "12/31/2024" ], "missing_count": 0, "missing_percent": 0.0, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json index 9c76a8c10753..78d6b236b6e7 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json @@ -1,6 +1,6 @@ { "title": "Expect column values to match strftime format", - "description": "Expect the column entries to be strings representing a date or time with a given format.\n\nExpectColumnValuesToMatchStrftimeFormat is a Column Map Expectation.\n\nColumn Map Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column and ask a yes/no question for every row in that column.\nBased on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid.\n\nArgs:\n column (str): The column name.\n strftime_format (str or SuiteParameterDict): A strftime format string to use for matching.\n\nOther Parameters:\n mostly (None or a float between 0 and 1): Successful if at least `mostly` fraction of values match the Expectation. For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly).\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Spark](https://docs.greatexpectations.io/docs/application_integration_support/)\n\n SQL data sources are not currently supported: strftime format tokens do not map cleanly onto the date-format models of SQL dialects.\n\nData Quality Issues:\n Validity\n\nExample Data:\n event_date\n 0 \"2024-01-15\"\n 1 \"2024-06-20\"\n 2 \"not-a-date\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 1,\n \"unexpected_percent\": 33.33333333333333,\n \"partial_unexpected_list\": [\n \"not-a-date\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 33.33333333333333,\n \"unexpected_percent_nonmissing\": 33.33333333333333\n },\n \"meta\": {},\n \"success\": false\n }\n\n Failing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%m/%d/%Y\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 3,\n \"unexpected_percent\": 100.0,\n \"partial_unexpected_list\": [\n \"2024-01-15\",\n \"2024-06-20\",\n \"not-a-date\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 100.0,\n \"unexpected_percent_nonmissing\": 100.0\n },\n \"meta\": {},\n \"success\": false\n }", + "description": "Expect the column entries to be strings representing a date or time with a given format.\n\nExpectColumnValuesToMatchStrftimeFormat is a Column Map Expectation.\n\nColumn Map Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column and ask a yes/no question for every row in that column.\nBased on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. SQL data sources are not currently supported: strftime format tokens do not map cleanly onto the date-format models of SQL dialects.\n\nArgs:\n column (str): The column name.\n strftime_format (str or SuiteParameterDict): A strftime format string to use for matching.\n\nOther Parameters:\n mostly (None or a float between 0 and 1): Successful if at least `mostly` fraction of values match the Expectation. For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). Default 1.\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Spark](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Validity\n\nExample Data:\n event_date invalid_date\n 0 \"2024-01-15\" \"01/15/2024\"\n 1 \"2024-06-20\" \"06/20/2024\"\n 2 \"2024-12-31\" \"12/31/2024\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 0,\n \"unexpected_percent\": 0.0,\n \"partial_unexpected_list\": [],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 0.0,\n \"unexpected_percent_nonmissing\": 0.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"invalid_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 3,\n \"unexpected_percent\": 100.0,\n \"partial_unexpected_list\": [\n \"01/15/2024\",\n \"06/20/2024\",\n \"12/31/2024\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 100.0,\n \"unexpected_percent_nonmissing\": 100.0\n },\n \"meta\": {},\n \"success\": false\n }", "type": "object", "properties": { "id": { From f026d35722e871f1d3cc8c3f3d7fc1a772b65724 Mon Sep 17 00:00:00 2001 From: nanjeshramesh Date: Tue, 4 Aug 2026 14:33:06 -0700 Subject: [PATCH 3/3] Actually remove the line continuation, not just relocate it My previous commit moved the SQL-unsupported sentence to the end of the intro paragraph but kept the backslash line-continuation between its two lines, reproducing the exact rendering issue that was flagged instead of fixing it. The surrounding sentences in that paragraph use plain lines with no continuation, so match that: each sentence is now its own line with no trailing backslash. Co-Authored-By: Claude Sonnet 5 --- .../core/expect_column_values_to_match_strftime_format.py | 5 ++--- .../schemas/ExpectColumnValuesToMatchStrftimeFormat.json | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py b/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py index acd84a946c91..95474667a7ae 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_strftime_format.py @@ -58,9 +58,8 @@ class ExpectColumnValuesToMatchStrftimeFormat(ColumnMapExpectation): Column Map Expectations are one of the most common types of Expectation. They are evaluated for a single column and ask a yes/no question for every row in that column. - Based on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. \ - SQL data sources are not currently supported: strftime format tokens do not map cleanly \ - onto the date-format models of SQL dialects. + Based on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. + SQL data sources are not currently supported: strftime format tokens do not map cleanly onto the date-format models of SQL dialects. Args: column (str): \ diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json index 78d6b236b6e7..af0d493f5019 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchStrftimeFormat.json @@ -1,6 +1,6 @@ { "title": "Expect column values to match strftime format", - "description": "Expect the column entries to be strings representing a date or time with a given format.\n\nExpectColumnValuesToMatchStrftimeFormat is a Column Map Expectation.\n\nColumn Map Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column and ask a yes/no question for every row in that column.\nBased on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid. SQL data sources are not currently supported: strftime format tokens do not map cleanly onto the date-format models of SQL dialects.\n\nArgs:\n column (str): The column name.\n strftime_format (str or SuiteParameterDict): A strftime format string to use for matching.\n\nOther Parameters:\n mostly (None or a float between 0 and 1): Successful if at least `mostly` fraction of values match the Expectation. For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). Default 1.\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Spark](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Validity\n\nExample Data:\n event_date invalid_date\n 0 \"2024-01-15\" \"01/15/2024\"\n 1 \"2024-06-20\" \"06/20/2024\"\n 2 \"2024-12-31\" \"12/31/2024\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 0,\n \"unexpected_percent\": 0.0,\n \"partial_unexpected_list\": [],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 0.0,\n \"unexpected_percent_nonmissing\": 0.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"invalid_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 3,\n \"unexpected_percent\": 100.0,\n \"partial_unexpected_list\": [\n \"01/15/2024\",\n \"06/20/2024\",\n \"12/31/2024\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 100.0,\n \"unexpected_percent_nonmissing\": 100.0\n },\n \"meta\": {},\n \"success\": false\n }", + "description": "Expect the column entries to be strings representing a date or time with a given format.\n\nExpectColumnValuesToMatchStrftimeFormat is a Column Map Expectation.\n\nColumn Map Expectations are one of the most common types of Expectation.\nThey are evaluated for a single column and ask a yes/no question for every row in that column.\nBased on the result, they then calculate the percentage of rows that gave a positive answer. If the percentage is high enough, the Expectation considers that data valid.\nSQL data sources are not currently supported: strftime format tokens do not map cleanly onto the date-format models of SQL dialects.\n\nArgs:\n column (str): The column name.\n strftime_format (str or SuiteParameterDict): A strftime format string to use for matching.\n\nOther Parameters:\n mostly (None or a float between 0 and 1): Successful if at least `mostly` fraction of values match the Expectation. For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). Default 1.\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, then catch exceptions and include them as part of the result object. For more detail, see [catch_exceptions](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#catch_exceptions).\n meta (dict or None): A JSON-serializable dictionary (nesting allowed) that will be included in the output without modification. For more detail, see [meta](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#meta).\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions. For more detail, see [failure severity](https://docs.greatexpectations.io/docs/cloud/expectations/expectations_overview/#failure-severity).\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\n Exact fields vary depending on the values passed to result_format, catch_exceptions, and meta.\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Spark](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Validity\n\nExample Data:\n event_date invalid_date\n 0 \"2024-01-15\" \"01/15/2024\"\n 1 \"2024-06-20\" \"06/20/2024\"\n 2 \"2024-12-31\" \"12/31/2024\"\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"event_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 0,\n \"unexpected_percent\": 0.0,\n \"partial_unexpected_list\": [],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 0.0,\n \"unexpected_percent_nonmissing\": 0.0\n },\n \"meta\": {},\n \"success\": true\n }\n\n Failing Case:\n Input:\n ExpectColumnValuesToMatchStrftimeFormat(\n column=\"invalid_date\",\n strftime_format=\"%Y-%m-%d\",\n )\n\n Output:\n {\n \"exception_info\": {\n \"raised_exception\": false,\n \"exception_traceback\": null,\n \"exception_message\": null\n },\n \"result\": {\n \"element_count\": 3,\n \"unexpected_count\": 3,\n \"unexpected_percent\": 100.0,\n \"partial_unexpected_list\": [\n \"01/15/2024\",\n \"06/20/2024\",\n \"12/31/2024\"\n ],\n \"missing_count\": 0,\n \"missing_percent\": 0.0,\n \"unexpected_percent_total\": 100.0,\n \"unexpected_percent_nonmissing\": 100.0\n },\n \"meta\": {},\n \"success\": false\n }", "type": "object", "properties": { "id": {