diff --git a/CHANGELOG.md b/CHANGELOG.md index 90899a8..ca28f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Add an `encoding` kwarg on `MetricReader.open` and `MetricWriter.open` (#43) - Transparent gzip/bz2/xz support via `xopen` (#44) - Infer the delimiter from the file extension in `MetricReader.open`/`MetricWriter.open`; pass `delimiter=` to override (#61) +- Restrict boolean parsing on `Metric` to a narrow, configurable token set (the `BoolTokens` converter), defaulting to fgpyo's `{true, t, 1}`/`{false, f, 0}` and rejecting anything else (#75) ### Changed diff --git a/fgmetric/_typing_extensions.py b/fgmetric/_typing_extensions.py index cea3ce6..7707d30 100644 --- a/fgmetric/_typing_extensions.py +++ b/fgmetric/_typing_extensions.py @@ -178,6 +178,68 @@ def is_list(annotation: TypeAnnotation | None) -> bool: return has_origin(annotation, list) +def is_bool(annotation: TypeAnnotation | None) -> bool: + """ + Check if a type annotation is `bool` or an Optional `bool`. + + Matches `bool`, `Optional[bool]`, and `bool | None`. Although `bool` is a subclass of `int`, + `int` is not matched. A union pairing `bool` with another non-None type (e.g. `bool | str`) is + not matched either, since it is not a pure boolean field. + + Examples: + >>> is_bool(bool) + True + >>> is_bool(bool | None) + True + >>> is_bool(int) + False + >>> is_bool(bool | str) + False + >>> is_bool(list[bool]) + False + """ + if annotation is None: + return False + + if is_optional(annotation): + annotation = unpack_optional(annotation) + + return annotation is bool + + +def is_bool_list(annotation: TypeAnnotation | None) -> bool: + """ + Check if a type annotation is a list with `bool` (or optional `bool`) elements. + + Matches `list[bool]` and `list[bool | None]`, including when the list itself is optional + (e.g. `list[bool] | None`). The element check reuses `is_bool`, so a list of any other element + type - including `int`, despite `bool` being a subclass - is not matched. + + Examples: + >>> is_bool_list(list[bool]) + True + >>> is_bool_list(list[bool | None]) + True + >>> is_bool_list(list[bool] | None) + True + >>> is_bool_list(list[int]) + False + >>> is_bool_list(bool) + False + """ + if annotation is None: + return False + + if is_optional(annotation): + annotation = unpack_optional(annotation) + + if not is_list(annotation): + return False + + args = get_args(annotation) + return len(args) == 1 and is_bool(args[0]) + + def is_counter(annotation: TypeAnnotation | None) -> bool: """ True if the type annotation is a Counter. diff --git a/fgmetric/collections/__init__.py b/fgmetric/collections/__init__.py deleted file mode 100644 index f3d9491..0000000 --- a/fgmetric/collections/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from fgmetric.collections._counter_pivot_table import CounterPivotTable -from fgmetric.collections._delimited_list import DelimitedList - -__all__ = [ - "CounterPivotTable", - "DelimitedList", -] diff --git a/fgmetric/converters/__init__.py b/fgmetric/converters/__init__.py new file mode 100644 index 0000000..7238a62 --- /dev/null +++ b/fgmetric/converters/__init__.py @@ -0,0 +1,11 @@ +from fgmetric.converters._bool_tokens import BoolTokens +from fgmetric.converters._counter_pivot_table import CounterPivotTable +from fgmetric.converters._delimited_list import DelimitedList +from fgmetric.converters._null_sentinels import NullSentinels + +__all__ = [ + "BoolTokens", + "CounterPivotTable", + "DelimitedList", + "NullSentinels", +] diff --git a/fgmetric/converters/_bool_tokens.py b/fgmetric/converters/_bool_tokens.py new file mode 100644 index 0000000..2c54554 --- /dev/null +++ b/fgmetric/converters/_bool_tokens.py @@ -0,0 +1,154 @@ +from typing import Any +from typing import ClassVar +from typing import final + +from pydantic import BaseModel +from pydantic import ValidationInfo +from pydantic import field_validator + +from fgmetric._typing_extensions import is_bool +from fgmetric._typing_extensions import is_bool_list + + +# NB: Inheriting from BaseModel is necessary to declare field validators on the mixin, and for the +# class-level initialization in `__pydantic_init_subclass__` to work. +class BoolTokens(BaseModel): + """ + Restrict boolean parsing to a narrow, configurable set of tokens. + + Pydantic accepts a broad set of strings as booleans (`true`/`t`/`yes`/`y`/`on`/`1` and their + false counterparts). With this mixin, fields annotated as `bool` or `bool | None` are instead + resolved from a narrow set of tokens *before* Pydantic's coercion runs: the incoming string + must match one of `true_tokens` or `false_tokens` (case-insensitively), or a `ValidationError` + is raised at the offending cell. `Metric` includes this behavior by default. + + The default token set mirrors `fgpyo.util.types.parse_bool`: `{"true", "t", "1"}` for `True` + and `{"false", "f", "0"}` for `False`. This is stricter than Pydantic, which would also accept + `yes`/`y`/`on`/`no`/`n`/`off`. Rejecting those guards against a misaligned or wrong-type column + being silently coerced, and gives tools ported from fgpyo identical accept/reject behavior. + + This mixin is read-side only; a `bool` still serializes to `"True"`/`"False"`. + + Tokens are matched case-insensitively. `true_tokens` and `false_tokens` must be disjoint; + declaring a token in both is rejected at class definition. + + Fields whose elements are bool - `list[bool]` and `list[bool | None]` - are tokenized per + element, so a list-of-bool column and a scalar `bool` column on the same model agree on what + counts as a valid boolean. Non-string values (and non-string list elements, such as a `None` + from an empty `list[bool | None]` cell) are left to Pydantic. + + Class Variables: + true_tokens: The strings resolved to `True`. Defaults to `frozenset({"true", "t", "1"})`. + false_tokens: The strings resolved to `False`. Defaults to `frozenset({"false", "f", "0"})`. + + Note: + `bool` is a subclass of `int`, but only fields annotated exactly as `bool` or `bool | None` + are affected; `int` fields are untouched. + + When combined with `NullSentinels`, the model-level null substitution runs first, so an + empty `bool | None` cell is converted to `None` before this validator sees it (provided + `""` is a configured null sentinel). Without such a sentinel, an empty cell on a `bool` + field is rejected, since `""` is in neither token set. + + Examples: + `Metric` parses bool fields from the default tokens out of the box: + + ```python + class MyMetric(Metric): + flag: bool + + MyMetric.model_validate({"flag": "1"}).flag # -> True + MyMetric.model_validate({"flag": "yes"}) # -> ValidationError + ``` + + Customize the accepted tokens: + + ```python + class MyMetric(Metric): + true_tokens = frozenset({"yes", "y"}) + false_tokens = frozenset({"no", "n"}) + flag: bool + + MyMetric.model_validate({"flag": "1"}) # -> ValidationError + MyMetric.model_validate({"flag": "yes"}).flag # -> True + ``` + """ + + # Public, user-specified token sets. + true_tokens: ClassVar[frozenset[str]] = frozenset({"true", "t", "1"}) + false_tokens: ClassVar[frozenset[str]] = frozenset({"false", "f", "0"}) + + # Internal: lowercase copies of the user-specified token sets, used for the actual matching. + _true_tokens: ClassVar[frozenset[str]] + _false_tokens: ClassVar[frozenset[str]] + + _bool_fieldnames: ClassVar[set[str]] + # NB: list-scoped because `list` is the only delimited collection today. When the + # `DelimitedCollection` work (#76) lands, generalize `is_bool_list` -> `is_bool_collection` + # (built on `is_collection`) so `set`/`frozenset`/`tuple` bool elements are tokenized too; the + # per-element logic below already works on the post-split list. Fixed-arity tuples (mixed + # element types) and dict bool values need their own handling. + _bool_list_fieldnames: ClassVar[set[str]] + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + """ + Record bool fields and the case-folded token sets used for matching. + + 1. The names of all fields annotated as `bool` or `bool | None` are stored in the private + `_bool_fieldnames` class variable, and the names of all fields whose elements are bool + (e.g. `list[bool]`, `list[bool | None]`) in `_bool_list_fieldnames`. + 2. `true_tokens` and `false_tokens` are case-folded into private sets for case-insensitive + matching, and validated to be disjoint. + """ + super().__pydantic_init_subclass__(**kwargs) + + cls._bool_fieldnames = { + name for name, info in cls.model_fields.items() if is_bool(info.annotation) + } + cls._bool_list_fieldnames = { + name for name, info in cls.model_fields.items() if is_bool_list(info.annotation) + } + cls._true_tokens = frozenset(token.casefold() for token in cls.true_tokens) + cls._false_tokens = frozenset(token.casefold() for token in cls.false_tokens) + cls._require_disjoint_tokens() + + @classmethod + def _require_disjoint_tokens(cls) -> None: + """Require that no token is configured as both true and false.""" + overlap = cls._true_tokens & cls._false_tokens + if overlap: + raise ValueError( + "true_tokens and false_tokens must be disjoint," + f" got overlapping tokens: {sorted(overlap)}" + ) + + @final + @field_validator("*", mode="before") + @classmethod + def _parse_bool_tokens(cls, value: Any, info: ValidationInfo) -> Any: + """Resolve string values on bool (and bool-list) fields from the configured tokens.""" + if isinstance(value, str) and info.field_name in cls._bool_fieldnames: + return cls._resolve_token(value) + + # On `Metric`, `DelimitedList` runs first and has already split a bool-list cell into a + # list of string elements by the time this validator sees it; resolve each element. Real + # `None` elements (e.g. an empty cell in a `list[bool | None]`) are left for Pydantic. + if isinstance(value, list) and info.field_name in cls._bool_list_fieldnames: + return [cls._resolve_token(el) if isinstance(el, str) else el for el in value] + + return value + + @classmethod + def _resolve_token(cls, value: str) -> bool: + """Resolve a single string token to `True`/`False`, or raise if it is not configured.""" + token = value.casefold() + if token in cls._true_tokens: + return True + elif token in cls._false_tokens: + return False + else: + raise ValueError( + f"Invalid boolean token {value!r}; expected one of" + f" {sorted(cls.true_tokens | cls.false_tokens)}" + ) diff --git a/fgmetric/collections/_counter_pivot_table.py b/fgmetric/converters/_counter_pivot_table.py similarity index 100% rename from fgmetric/collections/_counter_pivot_table.py rename to fgmetric/converters/_counter_pivot_table.py diff --git a/fgmetric/collections/_delimited_list.py b/fgmetric/converters/_delimited_list.py similarity index 100% rename from fgmetric/collections/_delimited_list.py rename to fgmetric/converters/_delimited_list.py diff --git a/fgmetric/converters/_null_sentinels.py b/fgmetric/converters/_null_sentinels.py new file mode 100644 index 0000000..eac402f --- /dev/null +++ b/fgmetric/converters/_null_sentinels.py @@ -0,0 +1,116 @@ +from typing import Any +from typing import ClassVar +from typing import final + +from pydantic import AliasChoices +from pydantic import BaseModel +from pydantic import model_validator +from pydantic.fields import FieldInfo + +from fgmetric._typing_extensions import is_optional + + +def _validation_keys(info: FieldInfo) -> set[str]: + """ + Return the string keys an input field may be supplied under. + + Covers the field's plain string `alias`/`validation_alias` and the string members of an + `AliasChoices`. `serialization_alias` is an output-only key, so it is excluded. `AliasPath` + aliases are path-shaped (for nested input) and do not apply to flat delimited rows, so they + are skipped. + """ + keys: set[str] = set() + if isinstance(info.alias, str): + keys.add(info.alias) + validation_alias = info.validation_alias + if isinstance(validation_alias, str): + keys.add(validation_alias) + elif isinstance(validation_alias, AliasChoices): + keys.update(choice for choice in validation_alias.choices if isinstance(choice, str)) + return keys + + +# NB: Inheriting from BaseModel is necessary to declare model validators on the mixin, and for the +# class-level initialization in `__pydantic_init_subclass__` to work. +class NullSentinels(BaseModel): + """ + Treat configured input strings as null on Optional fields. + + When this mixin is added to a model, the `null_sentinels` class variable declares the set of + input strings that represent null. Any field annotated as `T | None` whose incoming value is + one of the configured sentinels will be substituted with `None` before downstream field + validators run. + + The substitution is scoped to Optional fields. Non-Optional fields are not touched, which + keeps this mixin composable with field-specific handling (e.g., `DelimitedList`'s treatment + of `list[T]` fields). + + Class Variables: + null_sentinels: The set of input strings that should be treated as null on Optional + fields. Defaults to an empty set (no substitution). + + Examples: + Treat empty strings as null: + + ```python + class MyMetric(Metric): + null_sentinels = frozenset({""}) + name: str + count: int | None + + MyMetric.model_validate({"name": "foo", "count": ""}).count # -> None + ``` + + Treat multiple sentinels as null: + + ```python + class MyMetric(Metric): + null_sentinels = frozenset({"", "NA"}) + count: int | None + + MyMetric.model_validate({"count": "NA"}).count # -> None + ``` + """ + + null_sentinels: ClassVar[frozenset[str]] = frozenset() + _optional_field_keys: ClassVar[set[str]] + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + """ + Record the input keys of Optional-typed fields for null-sentinel substitution. + + The `mode="before"` validator sees input keyed as it arrives from the file, which is the + field's alias when one is set. So for each Optional field we register its canonical name + and any alias it may appear under, ensuring aliased fields are recognized. + """ + super().__pydantic_init_subclass__(**kwargs) + keys: set[str] = set() + for name, info in cls.model_fields.items(): + if not is_optional(info.annotation): + continue + keys.add(name) + keys |= _validation_keys(info) + cls._optional_field_keys = keys + + @final + @model_validator(mode="before") + @classmethod + def _substitute_null_sentinels(cls, data: Any) -> Any: + """Substitute incoming sentinel strings on Optional fields with `None`.""" + if not cls.null_sentinels: + return data + + if not isinstance(data, dict): + return data + + data = dict(data) + + for key in cls._optional_field_keys: + if key not in data: + continue + value = data[key] + if isinstance(value, str) and value in cls.null_sentinels: + data[key] = None + + return data diff --git a/fgmetric/metric.py b/fgmetric/metric.py index 15b66cc..efb126e 100644 --- a/fgmetric/metric.py +++ b/fgmetric/metric.py @@ -1,21 +1,23 @@ from abc import ABC from collections.abc import Sequence from pathlib import Path -from typing import Any +from typing import ClassVar from typing import Self from pydantic import BaseModel -from pydantic import model_validator -from fgmetric._typing_extensions import is_optional -from fgmetric.collections import CounterPivotTable -from fgmetric.collections import DelimitedList +from fgmetric.converters import BoolTokens +from fgmetric.converters import CounterPivotTable +from fgmetric.converters import DelimitedList +from fgmetric.converters import NullSentinels from fgmetric.metric_reader import MetricReader class Metric( DelimitedList, + BoolTokens, CounterPivotTable, + NullSentinels, BaseModel, ABC, ): @@ -32,15 +34,26 @@ class Metric( header of the file. Subclasses should define their fields using Pydantic field annotations. `Metric` includes the following custom serialization/deserialization behaviors: - 1. **Empty fields as None.** Any empty field in a file will be represented as `None` on the - deserialized model. + 1. **Null sentinels.** Empty strings in Optional fields are converted to `None` before field + validation. The sentinel values converted to `None` can be overridden by the + `null_sentinels` class variable. 2. **Delimited lists.** Any field typed as `list[T]` will be parsed from and serialized to a delimited string. The list delimiter may be controlled by the `collection_delimiter` class variable. + 3. **Boolean tokens.** Fields typed as `bool` (or `bool | None`), and the elements of + `list[bool]` fields, are parsed from a narrow, case-insensitive token set (`true`/`t`/`1`, + `false`/`f`/`0`), rejecting anything else rather than applying Pydantic's broader coercion. + The accepted tokens may be overridden by the `true_tokens`/`false_tokens` class variables. Class Variables: collection_delimiter: A single-character delimiter used to split and join `list` fields during serialization/deserialization. + null_sentinels: The set of input strings that should be treated as `None` on Optional + fields during validation. Defaults to `frozenset({""})`. + true_tokens: The set of input strings parsed as `True` on `bool` fields during validation. + Defaults to `frozenset({"true", "t", "1"})`. + false_tokens: The set of input strings parsed as `False` on `bool` fields during + validation. Defaults to `frozenset({"false", "f", "0"})`. Example: ```python @@ -55,6 +68,8 @@ class AlignmentMetric(Metric): ``` """ + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + @classmethod def read( cls, @@ -116,35 +131,6 @@ def read( ) as reader: return list(reader) - # NB: "Before" validators (mode="before") run before field validators such as - # `DelimitedList._split_lists()`. Empty strings in Optional fields will always be converted to - # `None` before any field validators. - # For example, for delimited list parsing: - # - When a field is defined as `list[T] | None`, this converts "" → None before _split_lists - # sees it. - # - When a field is defined as `list[T]`, "" passes through unchanged, then _split_lists - # converts "" → []. - @model_validator(mode="before") - @classmethod - def _empty_field_to_none(cls, data: Any) -> Any: - """Treat any empty fields as None if the field is typed as Optional.""" - if not isinstance(data, dict): - # short circuit - return data - - data = dict(data) - - for field, value in data.items(): - info = cls.model_fields.get(field) - if info is None: - # Skip fields that aren't defined on the model - let the validation handle it - continue - - if value == "" and is_optional(info.annotation): - data[field] = None - - return data - @classmethod def _header_fieldnames(cls) -> list[str]: """ diff --git a/tests/test_bool_tokens.py b/tests/test_bool_tokens.py new file mode 100644 index 0000000..b338a17 --- /dev/null +++ b/tests/test_bool_tokens.py @@ -0,0 +1,330 @@ +from pathlib import Path +from typing import ClassVar + +import pytest +from pydantic import Field +from pydantic import ValidationError + +from fgmetric import Metric +from fgmetric import MetricReader +from fgmetric.converters import BoolTokens +from fgmetric.converters import NullSentinels + + +@pytest.mark.parametrize("token", ["true", "t", "1", "TRUE", "True", "T"]) +def test_default_true_tokens(token: str) -> None: + """The default true tokens resolve to `True`, case-insensitively.""" + + class Model(BoolTokens): + flag: bool + + assert Model.model_validate({"flag": token}).flag is True + + +@pytest.mark.parametrize("token", ["false", "f", "0", "FALSE", "False", "F"]) +def test_default_false_tokens(token: str) -> None: + """The default false tokens resolve to `False`, case-insensitively.""" + + class Model(BoolTokens): + flag: bool + + assert Model.model_validate({"flag": token}).flag is False + + +@pytest.mark.parametrize("token", ["yes", "y", "on", "no", "n", "off"]) +def test_rejects_pydantic_extra_tokens(token: str) -> None: + """Tokens Pydantic would accept but fgpyo rejects raise a ValidationError.""" + + class Model(BoolTokens): + flag: bool + + with pytest.raises(ValidationError): + Model.model_validate({"flag": token}) + + +@pytest.mark.parametrize("token", ["", "2", "x", "maybe"]) +def test_rejects_arbitrary_strings(token: str) -> None: + """Strings outside the configured token sets raise a ValidationError.""" + + class Model(BoolTokens): + flag: bool + + with pytest.raises(ValidationError): + Model.model_validate({"flag": token}) + + +def test_optional_bool_resolves_tokens() -> None: + """A `bool | None` field still resolves configured string tokens.""" + + class Model(BoolTokens): + flag: bool | None + + assert Model.model_validate({"flag": "true"}).flag is True + assert Model.model_validate({"flag": "false"}).flag is False + + +def test_optional_bool_passes_through_none() -> None: + """A real `None` on a `bool | None` field is left untouched by the validator.""" + + class Model(BoolTokens): + flag: bool | None + + assert Model.model_validate({"flag": None}).flag is None + + +def test_real_bool_input_passes_through() -> None: + """Actual `bool` values (in-memory construction) are not affected.""" + + class Model(BoolTokens): + flag: bool + + assert Model(flag=True).flag is True + assert Model(flag=False).flag is False + + +def test_non_bool_string_fields_untouched() -> None: + """A `str` field whose value matches a bool token is not coerced.""" + + class Model(BoolTokens): + name: str + + assert Model.model_validate({"name": "true"}).name == "true" + + +def test_int_field_not_treated_as_bool() -> None: + """`int` fields are untouched even though `bool` is a subclass of `int`.""" + + class Model(BoolTokens): + count: int + + result = Model.model_validate({"count": "1"}) + assert result.count == 1 + assert type(result.count) is int # not coerced to bool (a subclass of int) + + +def test_bool_fieldnames_computed() -> None: + """Only `bool` / `bool | None` fields are recorded in `_bool_fieldnames`.""" + + class Model(BoolTokens): + flag: bool + maybe_flag: bool | None + name: str + count: int + mixed: bool | str # a mixed union is not a pure bool field + + assert Model._bool_fieldnames == {"flag", "maybe_flag"} + + +def test_bool_list_elements_resolve_tokens() -> None: + """Each element of a `list[bool]` field is resolved from the configured tokens.""" + + class Model(BoolTokens): + flags: list[bool] + + assert Model.model_validate({"flags": ["true", "f", "1", "0"]}).flags == [ + True, + False, + True, + False, + ] + + +@pytest.mark.parametrize("token", ["yes", "y", "on", "no", "n", "off", "x", ""]) +def test_bool_list_rejects_pydantic_extra_tokens(token: str) -> None: + """A list element Pydantic would coerce but fgpyo rejects raises a ValidationError.""" + + class Model(BoolTokens): + flags: list[bool] + + with pytest.raises(ValidationError): + Model.model_validate({"flags": ["true", token]}) + + +def test_optional_element_bool_list_preserves_none() -> None: + """A `list[bool | None]` resolves string elements but leaves real `None` elements alone.""" + + class Model(BoolTokens): + flags: list[bool | None] + + assert Model.model_validate({"flags": ["true", None, "false"]}).flags == [True, None, False] + + +def test_real_bool_list_input_passes_through() -> None: + """Actual `bool` elements (in-memory construction) are not affected.""" + + class Model(BoolTokens): + flags: list[bool] + + assert Model(flags=[True, False]).flags == [True, False] + + +def test_bool_list_fieldnames_computed() -> None: + """Only list fields with `bool` / `bool | None` elements are recorded as bool list fields.""" + + class Model(BoolTokens): + flags: list[bool] + maybe_flags: list[bool | None] + opt_flags: list[bool] | None + tags: list[int] + flag: bool + + assert Model._bool_list_fieldnames == {"flags", "maybe_flags", "opt_flags"} + assert Model._bool_fieldnames == {"flag"} + + +def test_metric_bool_list_is_strict() -> None: + """A `list[bool]` column on a `Metric` rejects the same tokens a scalar `bool` column does.""" + + class FlagMetric(Metric): + flags: list[bool] + + # The strict tokens resolve, splitting on the collection delimiter first. + assert FlagMetric.model_validate({"flags": "true,false,1,0"}).flags == [ + True, + False, + True, + False, + ] + # Tokens Pydantic would coerce are rejected, matching scalar `bool` behavior. + with pytest.raises(ValidationError): + FlagMetric.model_validate({"flags": "yes,on,no"}) + + +def test_metric_optional_element_bool_list() -> None: + """Empty cells in a `list[bool | None]` column become `None`; the rest are tokenized.""" + + class FlagMetric(Metric): + flags: list[bool | None] + + assert FlagMetric.model_validate({"flags": "1,,0"}).flags == [True, None, False] + + +def test_custom_tokens() -> None: + """Custom token sets replace the defaults.""" + + class Model(BoolTokens): + true_tokens: ClassVar[frozenset[str]] = frozenset({"yes", "y"}) + false_tokens: ClassVar[frozenset[str]] = frozenset({"no", "n"}) + + flag: bool + + assert Model.model_validate({"flag": "yes"}).flag is True + assert Model.model_validate({"flag": "n"}).flag is False + # The defaults are no longer accepted once overridden. + with pytest.raises(ValidationError): + Model.model_validate({"flag": "true"}) + + +def test_custom_tokens_matched_case_insensitively() -> None: + """Tokens configured in mixed case are still matched case-insensitively.""" + + class Model(BoolTokens): + true_tokens: ClassVar[frozenset[str]] = frozenset({"YES"}) + false_tokens: ClassVar[frozenset[str]] = frozenset({"NO"}) + + flag: bool + + assert Model.model_validate({"flag": "yes"}).flag is True + assert Model.model_validate({"flag": "no"}).flag is False + + +def test_aliased_bool_field_is_resolved() -> None: + """A bool field supplied under its alias is resolved from tokens.""" + + class Model(BoolTokens): + flag: bool = Field(alias="Flag") + + assert Model.model_validate({"Flag": "1"}).flag is True + + +def test_subclass_can_override_tokens() -> None: + """A subclass may override the token sets, leaving the parent unaffected.""" + + class Parent(BoolTokens): + flag: bool + + class Child(Parent): + true_tokens: ClassVar[frozenset[str]] = frozenset({"yes"}) + false_tokens: ClassVar[frozenset[str]] = frozenset({"no"}) + + assert Parent.model_validate({"flag": "true"}).flag is True + assert Child.model_validate({"flag": "yes"}).flag is True + with pytest.raises(ValidationError): + Child.model_validate({"flag": "true"}) + + +def test_overlapping_tokens_rejected_at_class_definition() -> None: + """Declaring a token in both `true_tokens` and `false_tokens` is an error.""" + with pytest.raises(ValueError, match="disjoint"): + + class Model(BoolTokens): + true_tokens: ClassVar[frozenset[str]] = frozenset({"1", "t"}) + false_tokens: ClassVar[frozenset[str]] = frozenset({"0", "t"}) + + flag: bool + + +def test_overlap_is_detected_case_insensitively() -> None: + """Overlap detection case-folds tokens, so `T` and `t` collide.""" + with pytest.raises(ValueError, match="disjoint"): + + class Model(BoolTokens): + true_tokens: ClassVar[frozenset[str]] = frozenset({"T"}) + false_tokens: ClassVar[frozenset[str]] = frozenset({"t"}) + + flag: bool + + +def test_metric_governs_bool_parsing_by_default(tmp_path: Path) -> None: + """`Metric` applies BoolTokens by default, governing bool parsing on the read path.""" + + class FlagMetric(Metric): + name: str + flag: bool + + fpath = tmp_path / "metrics.txt" + with fpath.open("w") as fout: + fout.write("name\tflag\n") + fout.write("ok\t1\n") + fout.write("nope\tyes\n") # rejected by the default tokens + + with MetricReader.open(FlagMetric, fpath) as reader: + rows = iter(reader) + assert next(rows).flag is True + with pytest.raises(ValidationError): + next(rows) + + +def test_metric_rejects_pydantic_tokens_by_default() -> None: + """A plain `Metric` rejects tokens Pydantic would accept (e.g. `yes`), by default.""" + + class FlagMetric(Metric): + flag: bool + + assert FlagMetric.model_validate({"flag": "true"}).flag is True + with pytest.raises(ValidationError): + FlagMetric.model_validate({"flag": "yes"}) + + +def test_interaction_with_null_sentinels() -> None: + """With `""` a null sentinel, an empty `bool | None` cell becomes None, not an error.""" + + class FlagMetric(Metric): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + flag: bool | None + + # Null substitution runs first, so "" -> None before BoolTokens sees it. + assert FlagMetric.model_validate({"flag": ""}).flag is None + # Non-empty tokens still resolve through BoolTokens. + assert FlagMetric.model_validate({"flag": "0"}).flag is False + + +def test_empty_bool_cell_rejected_without_sentinel() -> None: + """Without a null sentinel, an empty `bool` cell is rejected as an invalid token.""" + + class Model(BoolTokens, NullSentinels): + flag: bool + + with pytest.raises(ValidationError): + Model.model_validate({"flag": ""}) diff --git a/tests/test_null_sentinels.py b/tests/test_null_sentinels.py new file mode 100644 index 0000000..cb745d1 --- /dev/null +++ b/tests/test_null_sentinels.py @@ -0,0 +1,148 @@ +from typing import ClassVar + +import pytest +from pydantic import AliasChoices +from pydantic import ConfigDict +from pydantic import Field +from pydantic import ValidationError + +from fgmetric.converters import NullSentinels + + +def test_default_does_not_substitute() -> None: + """Default `null_sentinels = frozenset()` means no substitution happens.""" + + class Model(NullSentinels): + name: str + value: str | None + + result = Model.model_validate({"name": "foo", "value": ""}) + assert result.value == "" + + +def test_empty_string_treated_as_null() -> None: + """Empty strings become None on Optional fields when `""` is a configured sentinel.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + name: str + value: int | None + + result = Model.model_validate({"name": "foo", "value": ""}) + assert result.value is None + + +def test_aliased_optional_field_is_substituted() -> None: + """An Optional field supplied under its alias is recognized for substitution.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + value: int | None = Field(alias="Value") + + # Input is keyed by the alias, as a delimited-file row would be. + result = Model.model_validate({"Value": ""}) + assert result.value is None + + +def test_alias_choices_optional_field_is_substituted() -> None: + """An Optional field reachable via `AliasChoices` is substituted under any of its choices.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + value: int | None = Field(validation_alias=AliasChoices("v", "Value")) + + assert Model.model_validate({"v": ""}).value is None + assert Model.model_validate({"Value": ""}).value is None + + +def test_non_optional_fields_are_not_substituted() -> None: + """Non-Optional fields are not touched even when their input matches a sentinel.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({"NA"}) + + name: str + status: str + + result = Model.model_validate({"name": "foo", "status": "NA"}) + assert result.status == "NA" + + +def test_arbitrary_sentinels_are_supported() -> None: + """Sentinels other than `""` work, provided the field is Optional.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({"NA"}) + + status: str | None + + result = Model.model_validate({"status": "NA"}) + assert result.status is None + + +def test_multiple_sentinels() -> None: + """Multiple strings can be configured as null sentinels.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({"", "NA", "None"}) + + value: int | None + + assert Model.model_validate({"value": ""}).value is None + assert Model.model_validate({"value": "NA"}).value is None + assert Model.model_validate({"value": "None"}).value is None + + +def test_non_string_inputs_are_untouched() -> None: + """Non-string values are never substituted even if they would equal a sentinel.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + value: int | None + + result = Model.model_validate({"value": 0}) + assert result.value == 0 + + +def test_extra_fields_in_input_are_ignored() -> None: + """Keys in the input dict that aren't declared fields are not touched by the mixin.""" + + class Model(NullSentinels): + model_config = ConfigDict(extra="allow") + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + name: str + + result = Model.model_validate({"name": "foo", "extra_field": ""}) + assert result.model_extra == {"extra_field": ""} + + +def test_subclass_can_opt_out() -> None: + """A subclass can override `null_sentinels` to an empty frozenset to disable substitution.""" + + class Parent(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + value: str | None + + class Child(Parent): + null_sentinels: ClassVar[frozenset[str]] = frozenset() + + assert Parent.model_validate({"value": ""}).value is None + assert Child.model_validate({"value": ""}).value == "" + + +def test_sentinel_on_required_field_still_raises() -> None: + """A required field whose value matches a sentinel is not substituted, so pydantic raises.""" + + class Model(NullSentinels): + null_sentinels: ClassVar[frozenset[str]] = frozenset({""}) + + value: int + + with pytest.raises(ValidationError): + Model.model_validate({"value": ""}) diff --git a/tests/test_typing_extensions.py b/tests/test_typing_extensions.py index 6cdbe3a..a6ff170 100644 --- a/tests/test_typing_extensions.py +++ b/tests/test_typing_extensions.py @@ -6,6 +6,8 @@ from fgmetric._typing_extensions import TypeAnnotation from fgmetric._typing_extensions import has_optional_elements from fgmetric._typing_extensions import has_origin +from fgmetric._typing_extensions import is_bool +from fgmetric._typing_extensions import is_bool_list from fgmetric._typing_extensions import is_list from fgmetric._typing_extensions import is_optional from fgmetric._typing_extensions import unpack_optional @@ -77,6 +79,68 @@ def test_is_not_list(annotation: TypeAnnotation) -> None: assert not is_list(annotation) +@pytest.mark.parametrize( + "annotation", + [ + bool, + Optional[bool], + bool | None, + ], +) +def test_is_bool(annotation: TypeAnnotation) -> None: + """Should identify `bool`, even within an Optional.""" + assert is_bool(annotation) + + +@pytest.mark.parametrize( + "annotation", + [ + int, # `bool` is a subclass of `int`, but `int` is not `bool` + int | None, + str, + bool | str, # mixed union is not a pure bool field + list[bool], + bool | str | None, + ], +) +def test_is_not_bool(annotation: TypeAnnotation) -> None: + """Should reject types that are not exactly `bool` or `bool | None`.""" + assert not is_bool(annotation) + + +@pytest.mark.parametrize( + "annotation", + [ + list[bool], + list[bool | None], + list[Optional[bool]], + list[bool] | None, + list[bool | None] | None, + Optional[list[bool]], + Optional[list[bool | None]], + ], +) +def test_is_bool_list(annotation: TypeAnnotation) -> None: + """Should identify lists of `bool` / `bool | None`, even within an Optional.""" + assert is_bool_list(annotation) + + +@pytest.mark.parametrize( + "annotation", + [ + bool, # a scalar bool is not a list + bool | None, + list[int], # `bool` is a subclass of `int`, but a list of `int` is not a bool list + list[str], + list[bool | str], # a mixed element union is not a pure bool list + list[int] | None, + ], +) +def test_is_not_bool_list(annotation: TypeAnnotation) -> None: + """Should reject lists whose element type is not exactly `bool` or `bool | None`.""" + assert not is_bool_list(annotation) + + @pytest.mark.parametrize( "annotation,collection_type", [