feat: read metrics from a SQL query via from_sql (DuckDB) (#52)

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file.
 - Add an `encoding` kwarg on `MetricReader.open` and `MetricWriter.open` (#43)
 - Transparent gzip/bz2/xz support via `xopen` (#44)
 - Infer the delimiter from the file extension in `MetricReader.open`/`MetricWriter.open`; pass `delimiter=` to override (#61)
+- Read metrics from a SQL query via `MetricReader.from_sql`/`Metric.from_sql`, backed by the optional `duckdb` extra; queries any source DuckDB can read (Parquet/Arrow/JSON/CSV/SQLite/Postgres/S3) (#52)
 
 ### Changed
 

README.md

@@ -164,6 +164,32 @@ with MetricWriter.open(AlignmentMetric, "output.tsv.bz2") as writer:
     writer.writeall(metrics)
 ```
 
+### Reading from SQL (Parquet, Arrow, databases)
+
+With the optional `duckdb` extra (`pip install 'fgmetric[duckdb]'`), `MetricReader.from_sql` validates the rows of a SQL query as metrics. The query names its own source, so any backend [DuckDB](https://duckdb.org) can read works — Parquet, Arrow, JSON, CSV, SQLite, Postgres, and S3-hosted files — with no TSV intermediary:
+
+```python
+# A Parquet file (a transient in-memory connection is opened and closed for you):
+for metric in MetricReader.from_sql(AlignmentMetric, "SELECT * FROM 'metrics.parquet'"):
+    ...
+
+# Align a source's columns to your metric's fields with SQL `AS`:
+metrics = AlignmentMetric.from_sql("SELECT read AS read_name, mapq AS mapping_quality FROM tbl")
+```
+
+Unlike `open`, `from_sql` is not a context manager — it owns no file handle and returns a reader directly. `Metric.from_sql(query)` is the eager `list` form, mirroring `Metric.read(path)`.
+
+For backends that need engine setup (Postgres, custom secrets, `ATTACH`), pass a pre-configured connection; it is used as-is and never closed:
+
+```python
+import duckdb
+
+conn = duckdb.connect()
+conn.execute("INSTALL postgres; LOAD postgres")
+conn.execute("ATTACH 'dbname=metrics host=db' AS pg (TYPE postgres)")
+metrics = AlignmentMetric.from_sql("SELECT * FROM pg.alignment", connection=conn)
+```
+
 ### List Fields
 
 Fields typed as `list[T]` are automatically parsed from and serialized to delimited strings:

fgmetric/_duckdb.py

@@ -0,0 +1,48 @@
+from typing import TYPE_CHECKING
+from typing import Any
+
+if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection
+
+_MISSING_DUCKDB_MESSAGE = (
+    "DuckDB is required to read metrics from SQL. Install it with: pip install 'fgmetric[duckdb]'"
+)
+
+
+def query_rows(
+    query: str,
+    connection: "DuckDBPyConnection | None" = None,
+) -> list[dict[str, Any]]:
+    """
+    Run a DuckDB SQL query and return its rows as column-keyed dicts.
+
+    When `connection` is omitted, a transient in-memory DuckDB connection is opened for the
+    query and closed before returning. A supplied connection is used as-is and never closed,
+    so the caller may pre-configure extensions, secrets, or `ATTACH` and reuse it across calls.
+
+    Args:
+        query: A DuckDB SQL query. The query names its own source, e.g.
+            `SELECT * FROM 'data.parquet'`.
+        connection: An open DuckDB connection to run the query against. When `None`, a
+            transient in-memory connection is created and closed internally.
+
+    Returns:
+        One dict per result row, keyed by the query's output column names. Values are DuckDB's
+        native Python types (e.g. `int`, `float`, `Decimal`, `bool`, `None` for SQL NULL).
+
+    Raises:
+        ImportError: If the optional `duckdb` dependency is not installed.
+    """
+    try:
+        import duckdb
+    except ImportError as error:
+        raise ImportError(_MISSING_DUCKDB_MESSAGE) from error
+
+    conn = connection if connection is not None else duckdb.connect()
+    try:
+        relation = conn.sql(query)
+        columns = relation.columns
+        return [dict(zip(columns, row, strict=True)) for row in relation.fetchall()]
+    finally:
+        if connection is None:
+            conn.close()

fgmetric/metric.py

@@ -1,6 +1,7 @@
 from abc import ABC
 from collections.abc import Sequence
 from pathlib import Path
+from typing import TYPE_CHECKING
 from typing import Any
 from typing import Self
 
@@ -12,6 +13,9 @@
 from fgmetric.collections import DelimitedList
 from fgmetric.metric_reader import MetricReader
 
+if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection
+
 
 class Metric(
     DelimitedList,
@@ -116,6 +120,47 @@ def read(
         ) as reader:
             return list(reader)
 
+    @classmethod
+    def from_sql(
+        cls,
+        query: str,
+        *,
+        connection: "DuckDBPyConnection | None" = None,
+    ) -> list[Self]:
+        """
+        Read all Metric instances from the rows of a DuckDB SQL query.
+
+        Eager wrapper around `MetricReader.from_sql()`: the query runs and every row is
+        validated and collected into a list before this method returns. This is to
+        `from_sql` what `read` is to `MetricReader.open` — use `MetricReader.from_sql`
+        directly to iterate without materializing the full list.
+
+        The query names its own source, so any backend DuckDB can read works (Parquet, Arrow,
+        JSON, CSV, SQLite, Postgres, S3). See `MetricReader.from_sql()` for the connection and
+        column-naming behavior shared by both APIs. Requires the optional `duckdb` dependency:
+        `pip install 'fgmetric[duckdb]'`.
+
+        Args:
+            query: A DuckDB SQL query whose output columns match this metric class's fields.
+            connection: An open DuckDB connection. When `None`, a transient in-memory
+                connection is opened and closed internally. A supplied connection is never
+                closed.
+
+        Returns:
+            A list of instances of the calling Metric subclass, one per result row.
+
+        Raises:
+            ImportError: If the optional `duckdb` dependency is not installed.
+            ValidationError: If a row fails `Metric` validation.
+
+        Example:
+            ```python
+            metrics = AlignmentMetric.from_sql("SELECT * FROM 'metrics.parquet'")
+            print(f"read {len(metrics)} rows")
+            ```
+        """
+        return list(MetricReader.from_sql(cls, query, connection=connection))
+
     # NB: "Before" validators (mode="before") run before field validators such as
     # `DelimitedList._split_lists()`. Empty strings in Optional fields will always be converted to
     # `None` before any field validators.

fgmetric/metric_reader.py

@@ -1,34 +1,42 @@
 from collections.abc import Iterable
 from collections.abc import Iterator
+from collections.abc import Mapping
 from collections.abc import Sequence
 from contextlib import contextmanager
 from csv import DictReader
 from itertools import chain
 from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 from typing import Self
 
 from xopen import xopen
 
 from fgmetric._delimiter import infer_delimiter
+from fgmetric._duckdb import query_rows
 from fgmetric._paths import path_read_error
 
 if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection
+
     from fgmetric.metric import Metric
 
 
 class MetricReader[T: Metric]:
     """
-    Iterate `Metric` instances from a text IO source.
+    Iterate `Metric` instances from a parsed source.
 
     Constructed with any iterable of strings (file handle, StringIO, list of
     lines). The reader does not own the source; callers manage its lifecycle.
     Use the `open` classmethod to open and read a file in one step; unlike direct
     construction, `open` owns the file it opens and closes it on context exit.
+    Use `from_sql` to read from a DuckDB SQL query.
     """
 
     _metric_class: type[T]
-    _records: Iterator[dict[str, str | None]]
+    # Widened to Mapping[str, Any]: the SQL path yields already-typed values (int, float,
+    # Decimal, bool, None), not the strings the text/DictReader path produces.
+    _records: Iterator[Mapping[str, Any]]
 
     def __init__(
         self,
@@ -129,6 +137,78 @@ def open(
         with xopen(path, mode="rt", encoding=encoding) as handle:
             yield cls(metric_class, handle, delimiter, fieldnames)
 
+    @classmethod
+    def _from_records(
+        cls,
+        metric_class: type[T],
+        records: Iterator[Mapping[str, Any]],
+    ) -> Self:
+        """
+        Construct a reader directly from an iterator of column-keyed record dicts.
+
+        Shared tail for the text path (`__init__`, via `csv.DictReader`) and the SQL path
+        (`from_sql`). Bypasses `__init__` because the records are already dicts; there is no
+        text source to parse or header to detect.
+
+        Args:
+            metric_class: Metric class.
+            records: An iterator of column-keyed record dicts to validate on iteration.
+
+        Returns:
+            A `MetricReader` over the given records.
+        """
+        reader = cls.__new__(cls)
+        reader._metric_class = metric_class
+        reader._records = records
+        return reader
+
+    @classmethod
+    def from_sql(
+        cls,
+        metric_class: type[T],
+        query: str,
+        *,
+        connection: "DuckDBPyConnection | None" = None,
+    ) -> Self:
+        """
+        Read metrics from the rows returned by a DuckDB SQL query.
+
+        Unlike `open`, this is not a context manager: it owns no file handle and returns a
+        reader directly. The query names its own source, so any backend DuckDB can read works
+        — Parquet, Arrow, JSON, CSV, SQLite, Postgres, and S3-hosted files. Use SQL `AS` to
+        align a source's column names to the metric class's field names (or aliases).
+
+        Rows are fetched eagerly from DuckDB (so a transient connection's lifecycle is fully
+        contained), then validated lazily, one row per iteration, by `model_validate`.
+
+        Requires the optional `duckdb` dependency: `pip install 'fgmetric[duckdb]'`.
+
+        Args:
+            metric_class: Metric class.
+            query: A DuckDB SQL query whose output columns match the metric class's fields.
+            connection: An open DuckDB connection to run the query against. When `None` (the
+                default), a transient in-memory connection is opened and closed internally,
+                which covers local files and remote sources reachable with ambient
+                credentials. Pass a pre-configured connection to use extensions, secrets, or
+                `ATTACH`ed databases (e.g. Postgres); a supplied connection is never closed.
+
+        Returns:
+            A `MetricReader` over the query's rows.
+
+        Raises:
+            ImportError: If the optional `duckdb` dependency is not installed.
+            ValidationError: If a row fails `Metric` validation, e.g. a missing required
+                column or a value of the wrong type. Raised during iteration.
+
+        Example:
+            ```python
+            for metric in MetricReader.from_sql(AlignmentMetric, "SELECT * FROM 'm.parquet'"):
+                ...
+            ```
+        """
+        rows = query_rows(query, connection)
+        return cls._from_records(metric_class, iter(rows))
+
     def __iter__(self) -> Self:
         return self
 

pyproject.toml

@@ -43,6 +43,7 @@ dev = [
     "pytest-mock ~=3.14",
     "poethepoet ~=0.34",
     "pre-commit ~=4.0",
+    "duckdb >=1.0",
 ]
 ipython = [
     "ipython ~=9.2",
@@ -53,6 +54,9 @@ benchmark = [
     "pytest-benchmark ~=5.1.0",
     "fgpyo ~=1.2.0",
 ]
+duckdb = [
+    "duckdb >=1.0",
+]
 
 [tool.poe.tasks]
 fix-format = "ruff format"