diff --git a/CHANGELOG.md b/CHANGELOG.md index 514c637..9c22c21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Add an `encoding` kwarg on `MetricReader.open` and `MetricWriter.open` (#43) - Transparent gzip/bz2/xz support via `xopen` (#44) - Infer the delimiter from the file extension in `MetricReader.open`/`MetricWriter.open`; pass `delimiter=` to override (#61) +- Add append mode to `MetricWriter.open()` via `mode="a"`; appends to a file that has a matching header or that is headerless data validated against the model, raising only when the first row is neither (#47) ### Changed diff --git a/fgmetric/metric_writer.py b/fgmetric/metric_writer.py index 98ef4a0..78ec497 100644 --- a/fgmetric/metric_writer.py +++ b/fgmetric/metric_writer.py @@ -1,11 +1,14 @@ from collections.abc import Iterable from collections.abc import Iterator from contextlib import contextmanager +from csv import DictReader from csv import DictWriter from pathlib import Path +from typing import Literal from typing import Self from typing import TextIO +from pydantic import ValidationError from xopen import xopen from fgmetric._delimiter import infer_delimiter @@ -13,6 +16,111 @@ from fgmetric.metric import Metric +def _read_first_row( + path: Path | str, + delimiter: str, + encoding: str, +) -> list[str] | None: + """ + Return the first row of an existing, non-empty file as a list of cells, or `None`. + + Returns `None` when the file is missing or empty. Otherwise the first row is parsed with + `csv.DictReader` — the symmetric counterpart to the `DictWriter` used to write metrics — so it + is interpreted exactly as it was written. The row may be a header or a data record; the caller + decides which by comparing it to the expected column names and validating it against the model. + + Args: + path: Filesystem path to inspect. + delimiter: The delimiter used to parse the row. + encoding: The text encoding used to decode the file. + + Returns: + The parsed first row, or `None` if the file is missing or empty. + """ + file = Path(path) + # A zero-byte file is not a valid compressed stream; checking the size first avoids the + # bz2/xz decompressors raising EOFError when xopen tries to read an empty `.bz2`/`.xz`. + if not file.exists() or file.stat().st_size == 0: + return None + with xopen(path, mode="rt", encoding=encoding) as handle: + fieldnames = DictReader(handle, delimiter=delimiter).fieldnames + return None if fieldnames is None else list(fieldnames) + + +def _row_validates_as_record( + metric_class: type[Metric], + fieldnames: list[str], + row: list[str], +) -> bool: + """ + Whether `row`, zipped onto `fieldnames`, validates as a record of `metric_class`. + + The cells are mapped to `fieldnames` and validated with `model_validate`, the same path + `MetricReader` uses to parse a row. A row with the wrong number of cells cannot be a record, + so it is rejected without attempting validation. + + Args: + metric_class: Metric class to validate against. + fieldnames: The expected column names (the metric's header fields). + row: The parsed cells of the row to validate. + + Returns: + `True` if the row validates as a record, `False` otherwise. + """ + if len(row) != len(fieldnames): + return False + try: + metric_class.model_validate(dict(zip(fieldnames, row, strict=True))) + except ValidationError: + return False + return True + + +def _append_needs_header( + metric_class: type[Metric], + path: Path | str, + delimiter: str, + encoding: str, +) -> bool: + """ + Inspect a file opened for appending and report whether to write a header. + + Returns `True` when `path` is missing or empty — append-or-create writes a fresh header. + When `path` already has content, its first row is accepted if it either matches the expected + header or validates as a record of the metric class (a headerless data file); in both cases + `False` is returned so no header is written into the middle of the file. A `ValueError` is + raised only when the first row is neither. + + Args: + metric_class: Metric class whose header/records the file must be consistent with. + path: Filesystem path being opened for appending. + delimiter: The delimiter used to parse the first row. + encoding: The text encoding used to decode the file. + + Returns: + Whether a header row must be written. + + Raises: + ValueError: If `path` is non-empty and its first row is neither the expected header nor a + valid record. + """ + first_row = _read_first_row(path, delimiter, encoding) + if first_row is None: + return True + + expected = metric_class._header_fieldnames() + # A matching header, or a headerless file whose first row validates as a record: append after + # the existing content without writing a header. + if first_row == expected or _row_validates_as_record(metric_class, expected, first_row): + return False + + raise ValueError( + f"First row of {path} is neither a {metric_class.__name__} header nor a valid record.\n" + f" expected header: {expected}\n" + f" found: {first_row}" + ) + + class MetricWriter[T: Metric]: """ Write `Metric` instances to a text IO sink. @@ -32,15 +140,21 @@ def __init__( sink: TextIO, delimiter: str = "\t", lineterminator: str = "\n", + write_header: bool = True, ) -> None: """ - Initialize a new `MetricWriter` and write the header row to `sink`. + Initialize a new `MetricWriter`. + + By default the header row is written to `sink` immediately on construction. Pass + `write_header=False` to suppress it — e.g. when appending to a sink that already + contains a header. Args: metric_class: Metric class. sink: Writable text IO (e.g., file handle, StringIO) to write to. delimiter: The output file delimiter. lineterminator: The string used to terminate lines. + write_header: Whether to write the header row on construction. """ self._metric_class = metric_class self._writer = DictWriter( @@ -49,7 +163,8 @@ def __init__( delimiter=delimiter, lineterminator=lineterminator, ) - self._writer.writeheader() + if write_header: + self._writer.writeheader() @classmethod @contextmanager @@ -57,6 +172,7 @@ def open( cls, metric_class: type[T], path: Path | str, + mode: Literal["w", "a"] = "w", delimiter: str | None = None, lineterminator: str = "\n", encoding: str = "utf-8", @@ -72,11 +188,25 @@ def open( Compression is selected automatically based on the output file extension: plaintext, gzip (`.gz`), bzip2 (`.bz2`), or xz (`.xz`). - The header is written on context entry; the file is closed on context exit. + With `mode="w"` (the default) the file is truncated and the header row is written (refused + with `FileExistsError` if the file already exists and `overwrite` is `False`). With + `mode="a"` the file is opened for appending: if it is missing or empty the header is + written first (append-or-create); if it already has content, its first row is inspected and + rows are appended without writing a header when that row either matches the expected header + or validates as a record of the metric class (a headerless data file). A `ValueError` is + raised only when the first row is neither. The first row is parsed with the same `delimiter` + and `encoding`, so a foreign file written with a different delimiter or a BOM will fail to + validate (`MetricReader.open` defaults to `utf-8-sig`, which strips a BOM; the writer does + not). `MetricWriter` always terminates rows with `lineterminator`, so appending to a foreign + file whose last line lacks a trailing newline would concatenate. + + The file is opened lazily on context entry and closed on context exit. Args: metric_class: Metric class. path: Filesystem path to the output file. + mode: `"w"` to truncate and write, `"a"` to append. Append is non-destructive, so the + `overwrite` guard does not apply. delimiter: The output file delimiter. When `None` (the default), the delimiter is inferred from the file extension: `.csv` → comma; `.tsv`, `.txt`, `.tab`, or any extension ending in `metrics` → tab — ignoring any trailing compression @@ -95,22 +225,35 @@ def open( IsADirectoryError: If `path` is a directory. PermissionError: If `path` exists but is not writable, or if `path` cannot be created in its parent directory. - FileExistsError: If `path` already exists and `overwrite` is `False`. - ValueError: If `delimiter` is omitted and the delimiter cannot be inferred from - the file extension. + FileExistsError: If `path` already exists, `mode="w"`, and `overwrite` is `False`. + ValueError: If `delimiter` is omitted and cannot be inferred from the file + extension, or if `mode="a"` and the existing file's first row is neither the + expected header nor a valid record. Example: ```python with MetricWriter.open(AlignmentMetric, "metrics.txt") as writer: writer.writeall(metrics) + + with MetricWriter.open(AlignmentMetric, "metrics.txt", mode="a") as writer: + writer.writeall(more_metrics) ``` """ - if (error := path_write_error(path, overwrite=overwrite)) is not None: + if (error := path_write_error(path, overwrite=overwrite or mode == "a")) is not None: raise error if delimiter is None: delimiter = infer_delimiter(path) - with xopen(path, mode="wt", encoding=encoding) as handle: - yield cls(metric_class, handle, delimiter, lineterminator) + + xmode: Literal["wt", "at"] + if mode == "a": + xmode = "at" + write_header = _append_needs_header(metric_class, path, delimiter, encoding) + else: + xmode = "wt" + write_header = True + + with xopen(path, mode=xmode, encoding=encoding) as handle: + yield cls(metric_class, handle, delimiter, lineterminator, write_header=write_header) def write(self, metric: T) -> None: """ diff --git a/tests/test_metric_writer.py b/tests/test_metric_writer.py index 2cf4610..e79e315 100644 --- a/tests/test_metric_writer.py +++ b/tests/test_metric_writer.py @@ -12,6 +12,7 @@ from pytest_mock import MockerFixture from fgmetric import Metric +from fgmetric import MetricReader from fgmetric import MetricWriter from fgmetric import metric_writer @@ -261,3 +262,133 @@ def test_writer_open_overwrites_existing_file_when_overwrite_true(tmp_path: Path with MetricWriter.open(FakeMetric, p, overwrite=True) as writer: writer.write(FakeMetric(foo="abc", bar=1)) assert p.read_text() == "foo\tbar\nabc\t1\n" + + +def test_writer_constructor_can_skip_header() -> None: + """A writer constructed with write_header=False writes no header row.""" + sink = StringIO() + writer = MetricWriter(FakeMetric, sink, write_header=False) + writer.write(FakeMetric(foo="abc", bar=1)) + assert sink.getvalue() == "abc\t1\n" + + +def test_writer_append_skips_header_on_existing_file(tmp_path: Path) -> None: + """Appending to a file that already has a matching header does not rewrite it.""" + p = tmp_path / "out.tsv" + with MetricWriter.open(FakeMetric, p, mode="w") as writer: + writer.write(FakeMetric(foo="a", bar=1)) + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="b", bar=2)) + assert p.read_text() == "foo\tbar\na\t1\nb\t2\n" + + +def test_writer_append_writes_header_when_file_missing(tmp_path: Path) -> None: + """Append-or-create: appending to a missing file writes the header first.""" + p = tmp_path / "out.tsv" # does not exist + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="a", bar=1)) + assert p.read_text() == "foo\tbar\na\t1\n" + + +def test_writer_append_writes_header_when_file_empty(tmp_path: Path) -> None: + """Appending to an existing empty file writes the header first.""" + p = tmp_path / "out.tsv" + p.touch() + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="a", bar=1)) + assert p.read_text() == "foo\tbar\na\t1\n" + + +def test_writer_append_accepts_matching_header(tmp_path: Path) -> None: + """Appending to a file whose header matches the metric fields succeeds.""" + p = tmp_path / "out.tsv" + p.write_text("foo\tbar\n") + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="a", bar=1)) + assert p.read_text() == "foo\tbar\na\t1\n" + + +def test_writer_append_accepts_headerless_data_file(tmp_path: Path) -> None: + """Appending to a headerless file whose first row validates appends without a header.""" + p = tmp_path / "out.tsv" + p.write_text("abc\t1\ndef\t2\n") # data rows, no header + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="x", bar=9)) + # The file is recognized as headerless data, so no header is injected mid-file. + assert p.read_text() == "abc\t1\ndef\t2\nx\t9\n" + + +def test_writer_append_to_headerless_file_reads_back_with_fieldnames(tmp_path: Path) -> None: + """A headerless file stays headerless after appending and reads back with `fieldnames`.""" + p = tmp_path / "out.tsv" + p.write_text("abc\t1\n") + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="x", bar=9)) + with MetricReader.open(FakeMetric, p, fieldnames=["foo", "bar"]) as reader_: + assert list(reader_) == [FakeMetric(foo="abc", bar=1), FakeMetric(foo="x", bar=9)] + + +def test_writer_append_raises_when_first_row_neither_header_nor_record(tmp_path: Path) -> None: + """Appending raises when the first row is neither the header nor a valid record.""" + p = tmp_path / "out.tsv" + # `foo` accepts any string, but `bar` cannot parse as an int, so this is not a valid record; + # nor does it match the `foo\tbar` header. + p.write_text("abc\tnot_an_int\n") + with pytest.raises(ValueError, match="neither"): + with MetricWriter.open(FakeMetric, p, mode="a"): + pass + + +def test_writer_append_raises_when_first_row_has_wrong_column_count(tmp_path: Path) -> None: + """A first row with the wrong number of columns is neither header nor record, so it raises.""" + p = tmp_path / "out.tsv" + p.write_text("only_one_column\n") + with pytest.raises(ValueError, match="neither"): + with MetricWriter.open(FakeMetric, p, mode="a"): + pass + + +def test_writer_open_append_does_not_touch_file_until_enter(tmp_path: Path) -> None: + """open(mode="a") must not read or write the file until the context is entered.""" + p = tmp_path / "out.tsv" + p.write_text("foo\tbar\n") + MetricWriter.open(FakeMetric, p, mode="a") + assert p.read_text() == "foo\tbar\n" + + +@pytest.mark.parametrize("suffix", ["", ".gz", ".bz2", ".xz"]) +def test_writer_append_round_trips_across_formats(tmp_path: Path, suffix: str) -> None: + """Write then append then read back yields all rows, header-once, for every format.""" + p = tmp_path / f"out.tsv{suffix}" + with MetricWriter.open(FakeMetric, p, mode="w") as writer: + writer.write(FakeMetric(foo="a", bar=1)) + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="b", bar=2)) + + with MetricReader.open(FakeMetric, p) as reader_: + got = list(reader_) + + assert got == [FakeMetric(foo="a", bar=1), FakeMetric(foo="b", bar=2)] + + +@pytest.mark.parametrize("suffix", ["", ".gz", ".bz2", ".xz"]) +def test_writer_append_to_empty_file_writes_header_across_formats( + tmp_path: Path, suffix: str +) -> None: + """Append-or-create to an existing empty file writes the header first, for every format.""" + p = tmp_path / f"out.tsv{suffix}" + p.touch() # 0-byte file + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="a", bar=1)) + + with MetricReader.open(FakeMetric, p) as reader_: + assert list(reader_) == [FakeMetric(foo="a", bar=1)] + + +def test_writer_append_concatenates_when_last_line_lacks_newline(tmp_path: Path) -> None: + """Appending after a newline-less last line concatenates the new row, as documented.""" + p = tmp_path / "out.tsv" + p.write_text("foo\tbar\na\t1") # header + one row, no trailing newline + with MetricWriter.open(FakeMetric, p, mode="a") as writer: + writer.write(FakeMetric(foo="b", bar=2)) + assert p.read_text() == "foo\tbar\na\t1b\t2\n"