Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions src/gulliver/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,12 @@ def incremental(settings: Settings | None, sql: str, connection: str | None = No
if advice["strategy"] == "not_safe":
result["next_step"] = "Keep the full rebuild: " + "; ".join(advice["reasons"])
return result
if settings is not None:
refusal, notes = _check_incremental_names(settings, connection, rendered, advice, time_column, unique_key)
if refusal is not None:
return refusal
if notes:
result["notes"] = [*advice.get("notes", []), *notes]
if settings is not None and estimate_savings and advice.get("time_column_source"):
try:
result["savings"] = _savings(settings, connection, advice, dialect)
Expand All @@ -393,6 +399,57 @@ def incremental(settings: Settings | None, sql: str, connection: str | None = No
return result


def _check_incremental_names(settings: Settings, connection: str | None, sql: str, advice: dict[str, Any],
time_column: str | None, unique_key: list[str] | None
) -> tuple[dict[str, Any] | None, list[str]]:
"""Refuse a --time-column or --unique-key that the database does not have.

Reads only column names (LIMIT 0) through the guarded path. When the columns cannot be read (a table
that is not there, a model that does not run as plain SQL), the check is skipped and a note says so.
"""
keys = [k for k in (unique_key or []) if k]
source = advice.get("time_column_source")
check_time = bool(time_column) and bool(source)
if not check_time and not keys:
return None, []
notes: list[str] = []

def columns_of(engine: Engine, conn_name: str, query: str) -> list[str] | None:
try:
return _gated_run(settings, conn_name, engine, f"SELECT * FROM {query} LIMIT 0", "incremental",
preview_rows=0, ignore_rules=frozenset({"GV004"}))["columns"]
except (Blocked, EngineError):
return None

try:
with _engine(settings, connection) as (conn, engine):
if check_time:
source_cols = columns_of(engine, conn.name, source)
wanted = str(time_column).split(".")[-1]
if source_cols is None:
notes.append(f"Could not read the columns of {source}, so --time-column was not checked.")
elif wanted.lower() not in {c.lower() for c in source_cols}:
stamps = [c for c in source_cols if incr.TIME_NAME.search(c)]
hint = (f"Timestamp-like columns of {source}: {', '.join(stamps)}." if stamps
else f"Columns of {source}: {', '.join(source_cols)}.")
return _error(f"Time column '{wanted}' is not a column of {source}.",
f"{hint} Pass --time-column with one of them."), notes
if keys:
model_cols = columns_of(engine, conn.name, f"({sql.strip().rstrip(';')}) AS gv_model")
if model_cols is None:
notes.append("Could not run the model to read its columns, so --unique-key was not checked.")
else:
have = {c.lower() for c in model_cols}
missing = [k for k in keys if k.lower() not in have]
if missing:
return _error(f"Unique key column(s) not returned by the model: {', '.join(missing)}",
f"The model returns: {', '.join(model_cols)}. Pass --unique-key with "
"names from that list."), notes
except (EngineError, ConfigError):
notes.append("Could not connect to read column names, so --time-column and --unique-key were not checked.")
return None, notes


def _savings(settings: Settings, connection: str | None, advice: dict[str, Any], dialect: str) -> dict[str, Any]:
source, tcol = advice["time_column_source"], advice["time_column"]
step = incr.parse_period(advice["period"]) * (1 + int(advice.get("lookback_periods") or 0))
Expand Down
45 changes: 45 additions & 0 deletions tests/test_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,51 @@ def test_incremental_append_dbt_uses_this(settings):
assert "unique_key='event_id'" in r["dbt_model"]


def test_incremental_refuses_an_unknown_time_column(settings):
r = features.incremental(settings, "select user_id, count(*) as n from events group by 1",
time_column="nosuch")
assert r["status"] == "error"
assert r["error"] == "Time column 'nosuch' is not a column of events."
assert "ts" in r["next_step"]
assert "incremental_sql" not in r


def test_incremental_refuses_an_unknown_unique_key(settings):
r = features.incremental(settings, "select user_id, max(ts) as last_ts, count(*) as n from events group by 1",
time_column="ts", unique_key=["user_id", "nosuch"])
assert r["status"] == "error"
assert r["error"] == "Unique key column(s) not returned by the model: nosuch"
assert "user_id, last_ts, n" in r["next_step"]


def test_incremental_names_are_compared_case_insensitively(settings):
sql = "select user_id, max(ts) as last_ts, count(*) as n from events group by 1"
r = features.incremental(settings, sql, time_column="TS", unique_key=["USER_ID"], estimate_savings=False)
assert r["status"] == "ok"
assert r["strategy"] == "merge_recompute"


def test_incremental_valid_names_give_the_same_advice_as_before(settings):
sql = "select user_id, max(ts) as last_ts, count(*) as n from events group by 1"
checked = features.incremental(settings, sql, time_column="ts", unique_key=["user_id"], estimate_savings=False)
offline = features.incremental(None, sql, time_column="ts", unique_key=["user_id"], estimate_savings=False)
assert checked == offline


def test_incremental_without_a_connection_cannot_check_names():
r = features.incremental(None, "select user_id, count(*) as n from events group by 1",
time_column="nosuch", unique_key=["nosuch"])
assert r["status"] == "ok"
assert "dbt_model" in r


def test_incremental_notes_when_the_model_cannot_be_run_to_read_its_columns(settings):
r = features.incremental(settings, "select user_id, count(*) as n from events_that_do_not_exist group by 1",
time_column="ts", unique_key=["nosuch"], estimate_savings=False)
assert r["status"] == "ok"
assert any("--unique-key was not checked" in n for n in r["notes"])


def test_parse_period():
assert incremental.parse_period("24h").total_seconds() == 86400
with pytest.raises(ValueError):
Expand Down
Loading