From 676dbd5783f19a2de1717f61c15b548b1de6e9fc Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Tue, 6 Oct 2026 17:49:08 +0200 Subject: [PATCH 1/3] Refactor expression matching to separate module --- django_ormql/expressions.py | 856 ++++++++++++++++++++++++++++++++++++ django_ormql/query.py | 613 +------------------------- 2 files changed, 864 insertions(+), 605 deletions(-) create mode 100644 django_ormql/expressions.py diff --git a/django_ormql/expressions.py b/django_ormql/expressions.py new file mode 100644 index 0000000..492123b --- /dev/null +++ b/django_ormql/expressions.py @@ -0,0 +1,856 @@ +from datetime import datetime, date + +from django.db import models +from django.db.models import ( + BooleanField, + ExpressionWrapper, + F, + OuterRef, + Q, + Value, + aggregates, + functions, + lookups, +) +from django.db.models.fields.json import KeyTextTransform, KeyTransform +from sqlglot import expressions + +from . import db_func +from .db_func import _patch_func +from .exceptions import QueryError, QueryNotSupported + + +class Expression: + node_class = None + sql_type = None + + def matches(self, expression): + return isinstance(expression, self.node_class) + + def to_django(self, expression, recurse, **kwargs): + pass + + def to_name(self, expression): + return expression.sql() + + +class FuncExpression(Expression): + node_class = expressions.Anonymous + func_name = None + + def matches(self, expression): + return super().matches(expression) and expression.this.lower() == self.func_name + + +_expressions = [] + + +def register(cls): + _expressions.append(cls()) + return cls + + +def expression_to_django(expression, recurse, **kwargs): + for e in _expressions: + if e.matches(expression): + return e.to_django(expression, recurse, **kwargs) + raise QueryNotSupported(f"Unsupported expression: {expression.sql()}") + + +def expression_to_name(expression): + for e in _expressions: + if e.matches(expression): + return e.to_name(expression) + return expression.sql() + + +types = { + expressions.DataType.Type.BIGDECIMAL: models.DecimalField( + max_digits=20, decimal_places=2 + ), # TODO variable? + expressions.DataType.Type.DECIMAL: models.DecimalField( + max_digits=20, decimal_places=2 + ), # TODO variable? + expressions.DataType.Type.BIGINT: models.BigIntegerField(), + expressions.DataType.Type.BIGSERIAL: models.BigIntegerField(), + expressions.DataType.Type.INT: models.IntegerField(), + expressions.DataType.Type.BOOLEAN: models.BooleanField(), + expressions.DataType.Type.JSON: models.JSONField(), + expressions.DataType.Type.JSONB: models.JSONField(), + expressions.DataType.Type.DOUBLE: models.FloatField(), + expressions.DataType.Type.FLOAT: models.FloatField(), + expressions.DataType.Type.TEXT: models.TextField(), + expressions.DataType.Type.TIME: models.TimeField(), + expressions.DataType.Type.TIMESTAMPTZ: models.DateTimeField(), + expressions.DataType.Type.DATETIME: models.DateTimeField(), + expressions.DataType.Type.DATE: models.DateField(), +} + + +# Maps Django's Field.get_internal_type() to the ORMQL sql_type vocabulary +# emitted by BaseColumn.sql_type subclasses (see columns.py). Keeping one +# vocabulary means /tables/ introspection and per-query result metadata speak +# the same language. +INTERNAL_TYPE_TO_SQL_TYPE = { + "DateField": "DATE", + "DateTimeField": "DATETIME", + "TimeField": "TIME", + "DurationField": "DURATION", + "IntegerField": "INT", + "BigIntegerField": "INT", + "SmallIntegerField": "INT", + "PositiveIntegerField": "INT", + "PositiveSmallIntegerField": "INT", + "PositiveBigIntegerField": "INT", + "AutoField": "INT", + "BigAutoField": "INT", + "SmallAutoField": "INT", + "DecimalField": "DECIMAL", + "FloatField": "FLOAT", + "BooleanField": "BOOLEAN", + "JSONField": "JSONB", + "CharField": "TEXT", + "TextField": "TEXT", + "EmailField": "TEXT", + "URLField": "TEXT", + "SlugField": "TEXT", + "UUIDField": "TEXT", + "GenericIPAddressField": "TEXT", + "FileField": "TEXT", + "FilePathField": "TEXT", + "ImageField": "TEXT", + "BinaryField": "TEXT", +} + + +def _describe_expression(expr, query=None): + """Extract (sql_type, nullable, hint) from a Django expression's output_field. + Hint is dependent on type and currently only available for truncated datetimes. + + Aggregates and Cast() / typed Func() have `.output_field` directly. Plain + F() references only know their field once resolved against a query; when + `query` is supplied, we resolve on a cloned query (so we don't mutate the + original with extra joins) and read `output_field` off the resolved node. + + Returns ("", None, None) for anything we can't pin down — the frontend treats + that as "unknown, render opaquely" which matches the pre-metadata + behavior. + """ + out = None + try: + out = expr.output_field + except: # noqa + out = None + if out is None and query is not None: + try: + resolved = expr.resolve_expression(query=query.chain(), allow_joins=True) + out = resolved.output_field + except: # noqa + out = None + if out is None: + return "", None + try: + internal = out.get_internal_type() + except: # noqa + return "", None + sql_type = INTERNAL_TYPE_TO_SQL_TYPE.get(internal, "") + nullable = getattr(out, "null", None) + hint = None + + if isinstance(expr, functions.datetime.TruncBase): + hint = {"truncated": expr.kind} + + return sql_type, nullable, hint + +def _to_column_path(expression): + """ + Hack an expression like part1.part2.part3.part4.part5 to [part1, part2, part3, part4, part5] + """ + if isinstance(expression, expressions.Dot): + return [ + *_to_column_path(expression.this), + expression.expression.this, + ] + elif isinstance(expression, expressions.Column): + return [ + x.this + for x in [ + expression.args.get("catalog"), + expression.args.get("db"), + expression.args.get("table"), + expression.args.get("this"), + ] + if x + ] + else: + raise TypeError("Invalid type") + + +@register +class Column(Expression): + def matches(self, expression): + return isinstance(expression, (expressions.Column, expressions.Dot)) + + def to_django(self, expression, recurse, table, aggregate_names, **kwargs): + cp = _to_column_path(expression) + if len(cp) == 1 and aggregate_names and cp[0] in aggregate_names: + return F(aggregate_names[cp[0]]) + return table.resolve_column_path(cp) + + def to_name(self, expression): + return ".".join(_to_column_path(expression)) + + +@register +class Alias(Expression): + node_class = expressions.Alias + + def to_django(self, expression, recurse, **kwargs): + return recurse(expression.this, **kwargs) + + def to_name(self, expression): + return expression.output_name + + +@register +class Literal(Expression): + node_class = expressions.Literal + + def to_django(self, expression, recurse, **kwargs): + return Value(expression.to_py()) + + def to_name(self, expression): + return str(expression.this) + + +@register +class Boolean(Expression): + node_class = expressions.Boolean + sql_type = "BOOLEAN" + + def to_django(self, expression, recurse, **kwargs): + return Value(expression.this) + + +@register +class Star(Expression): + node_class = expressions.Star + sql_type = "TEXT" + + def to_django(self, expression, recurse, **kwargs): + return "*" + + +@register +class Cast(Expression): + node_class = expressions.Cast + + def to_django(self, expression, recurse, **kwargs): + return models.functions.Cast( + recurse(expression.this, **kwargs), + output_field=types[expression.to.this], + ) + + +@register +class Extract(Expression): + node_class = expressions.Extract + + def to_django(self, expression, recurse, timezone, **kwargs): + if isinstance(expression.this, expressions.Var): + lookup_name = expression.this.this.lower() + else: + lookup_name = expression.this.to_py() + if lookup_name not in ( + "year", + "iso_year", + "quarter", + "month", + "day", + "week", + "week_day", + "iso_week_day", + "hour", + "minute", + "second", + ): + raise QueryNotSupported(f"Unsupported extract value '{lookup_name}'") + return functions.Extract( + recurse(expression.expression, **kwargs), + lookup_name=lookup_name, + tzinfo=timezone, + ) + + +@register +class Outer(FuncExpression): + func_name = "outer" + + def to_django(self, expression, recurse, **kwargs): + parent_table_stack = kwargs.get("parent_table_stack", []) + + def _resolve(e, parent_stack, depth): + if isinstance(e, expressions.Anonymous) and e.this.lower() == "outer": + if not parent_stack: + raise QueryError("OUTER nested too far") + return _resolve(e.expressions[0], parent_stack[:-1], depth + 1) + elif isinstance(e, (expressions.Column, expressions.Dot)): + if not parent_stack: + raise QueryError("OUTER nested too far") + return _to_column_path(e), parent_stack[-1], depth + else: + raise QueryNotSupported("Invalid argument to OUTER()") + + cp, lookup_table, depth = _resolve( + expression.expressions[0], parent_table_stack, 1 + ) + p = lookup_table.resolve_column_path(cp) + if isinstance(p, F): + p = p.name + else: + raise QueryNotSupported(f"Cannot use '{cp}' in OUTER()") + for i in range(depth): + p = OuterRef(p) + return p + + +@register +class Datetrunc(FuncExpression): + func_name = "datetrunc" + + def to_django(self, expression, recurse, timezone, **kwargs): + if len(expression.expressions) != 2: + raise QueryError("Function datetrunc takes exactly two arguments") + try: + lookup_name = expression.expressions[0].to_py() + if lookup_name not in ( + "year", + "quarter", + "month", + "day", + "week", + "hour", + "minute", + "second", + ): + raise QueryNotSupported(f"Unsupported truncation type '{lookup_name}'") + except ValueError: + raise QueryNotSupported("Unsupported truncation type") + return functions.Trunc( + recurse(expression.expressions[1], **kwargs), + lookup_name, + tzinfo=timezone, + ) + + +function_nodes = { + expressions.Coalesce: _patch_func(functions.Coalesce), + expressions.Concat: db_func.PatchedConcatPair, + expressions.Greatest: _patch_func(functions.Greatest), + expressions.Least: _patch_func(functions.Least), + expressions.Abs: _patch_func(functions.Abs), + expressions.Ceil: _patch_func(functions.Ceil), + expressions.Floor: _patch_func(functions.Floor), + expressions.Mod: _patch_func(functions.Mod), + expressions.Left: _patch_func(functions.Left), + expressions.Right: _patch_func(functions.Right), + expressions.Length: _patch_func(functions.Length), + expressions.Lower: _patch_func(functions.Lower), + expressions.Upper: _patch_func(functions.Upper), + expressions.SubstringIndex: _patch_func(functions.StrIndex), +} + + +@register +class Function(Expression): + def matches(self, expression): + return type(expression) in function_nodes + + def to_django(self, expression, recurse, **kwargs): + if expression.args.get("this"): + args = [recurse(expression.this, **kwargs)] + else: + args = [] + if expression.args.get("expression"): + args += [recurse(expression.expression, **kwargs)] + args += [recurse(e, **kwargs) for e in expression.expressions] + cls = function_nodes[type(expression)] + if (cls.arity and cls.arity != len(args)) or any( + v is not None + and k + not in ( + "this", + "expression", + "expressions", + "ignore_nulls", + "safe", + "coalesce", + ) + for k, v in expression.args.items() + ): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + return cls(*args) + + +@register +class Round(Expression): + node_class = expressions.Round + + def to_django(self, expression, recurse, **kwargs): + args = [ + recurse(expression.this, **kwargs), + ] + if expression.args.get("decimals"): + args.append(recurse(expression.args["decimals"], **kwargs)) + if any( + v is not None and k not in ("this", "decimals") + for k, v in expression.args.items() + ): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + return functions.Round(*args) + + +@register +class Pad(Expression): + node_class = expressions.Pad + + def to_django(self, expression, recurse, **kwargs): + args = [ + recurse(expression.this, **kwargs), + recurse(expression.expression, **kwargs), + ] + if expression.args.get("fill_pattern"): + args.append(recurse(expression.args["fill_pattern"], **kwargs)) + if any( + v is not None and k not in ("this", "expression", "fill_pattern", "is_left") + for k, v in expression.args.items() + ): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + if expression.args["is_left"]: + return functions.LPad(*args) + else: + return functions.RPad(*args) + + +@register +class StrPosition(Expression): + node_class = expressions.StrPosition + + def to_django(self, expression, recurse, **kwargs): + if not expression.args.get("substr"): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + args = [ + recurse(expression.this, **kwargs), + recurse(expression.args["substr"], **kwargs), + ] + if any( + v is not None and k not in ("this", "substr") + for k, v in expression.args.items() + ): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + return functions.StrIndex(*args) + + +@register +class Substring(Expression): + node_class = expressions.Substring + + def to_django(self, expression, recurse, **kwargs): + if not expression.args.get("start"): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + args = [ + recurse(expression.this, **kwargs), + recurse(expression.args["start"], **kwargs), + ] + if expression.args.get("length"): + args.append(recurse(expression.args["length"], **kwargs)) + if any( + v is not None and k not in ("this", "start", "length") + for k, v in expression.args.items() + ): + raise QueryNotSupported( + f"Wrong number of arguments for function {expression.sql()}" + ) + return functions.Substr(*args) + + +@register +class Replace(Expression): + node_class = expressions.Replace + + def to_django(self, expression, recurse, **kwargs): + args = [ + recurse(expression.this, **kwargs), + recurse(expression.expression, **kwargs), + ] + if expression.args.get("replacement"): + args.append(recurse(expression.args["replacement"], **kwargs)) + return functions.Replace(*args) + + +@register +class DPipe(Expression): + node_class = expressions.DPipe + + def to_django(self, expression, recurse, **kwargs): + return functions.Concat( + recurse(expression.this, **kwargs), + recurse(expression.expression, **kwargs), + ) + + +aggregate_nodes = { + expressions.Avg: aggregates.Avg, + expressions.Count: aggregates.Count, + expressions.Max: aggregates.Max, + expressions.Min: aggregates.Min, + expressions.Stddev: aggregates.StdDev, + expressions.Variance: aggregates.Variance, + expressions.Sum: aggregates.Sum, +} + + +@register +class Filter(Expression): + node_class = expressions.Filter + + def matches(self, node): + return super().matches(node) and type(node.this) in aggregate_nodes + + def to_django(self, expression, recurse, **kwargs): + if isinstance(expression.this.this, expressions.Distinct): + args = [recurse(e, **kwargs) for e in expression.this.this.expressions] + distinct = True + else: + args = [recurse(expression.this.this, **kwargs)] + distinct = False + if len(args) > 1: + raise QueryNotSupported( + "Multiple arguments to aggregate expression not supported" + ) + return aggregate_nodes[type(expression.this)]( + *args, + distinct=distinct, + filter=recurse(expression.expression.this, **kwargs), + ) + + +@register +class Aggregate(Expression): + def matches(self, expression): + return type(expression) in aggregate_nodes + + def to_django(self, expression, recurse, **kwargs): + if isinstance(expression.this, expressions.Distinct): + args = [recurse(e, **kwargs) for e in expression.this.expressions] + distinct = True + else: + args = [recurse(expression.this, **kwargs)] + distinct = False + if len(args) > 1: + raise QueryNotSupported( + "Multiple arguments to aggregate expression not supported" + ) + cls = aggregate_nodes[type(expression)] + return cls(*args, distinct=distinct) + + +math_binary_nodes = { + expressions.Mul: db_func.Mul, + expressions.Add: db_func.Add, + expressions.Sub: db_func.Sub, + expressions.Div: db_func.Div, + expressions.Mod: db_func.Mod, +} + + +@register +class MathBinary(Expression): + def matches(self, expression): + return type(expression) in math_binary_nodes + + def to_django(self, expression, recurse, **kwargs): + lhs = recurse(expression.left, **kwargs) + rhs = recurse(expression.right, **kwargs) + return math_binary_nodes[type(expression)]( + lhs, + rhs, + ) + + +@register +class Order(Expression): + node_class = expressions.Order + + def to_django(self, expression, recurse, **kwargs): + raise QueryNotSupported("ORDER not supported in expression") + + +@register +class Null(Expression): + node_class = expressions.Null + + def to_django(self, expression, recurse, **kwargs): + # TODO do we need to guess output_field better? + return Value(None, output_field=models.TextField(null=True)) + + +@register +class NullSafeEQ(Expression): + node_class = expressions.NullSafeEQ + + def to_django(self, expression, recurse, **kwargs): + raise QueryNotSupported("IS (NOT) DISTINCT not supported") + + +@register +class NullSafeNEQ(Expression): + node_class = expressions.NullSafeNEQ + + def to_django(self, expression, recurse, **kwargs): + raise QueryNotSupported("IS (NOT) DISTINCT not supported") + + +@register +class Paren(Expression): + node_class = expressions.Paren + + def to_django(self, expression, recurse, **kwargs): + return recurse(expression.this, **kwargs) + + +@register +class Neg(Expression): + node_class = expressions.Neg + + def to_django(self, expression, recurse, **kwargs): + return -recurse(expression.this, **kwargs) + + +@register +class Bitwise(Expression): + def matches(self, expression): + return isinstance( + expression, + ( + expressions.BitwiseNot, + expressions.BitwiseOr, + expressions.BitwiseXor, + expressions.BitwiseAnd, + expressions.BitwiseCount, + expressions.BitwiseLeftShift, + expressions.BitwiseRightShift, + ), + ) + + def to_django(self, expression, recurse, **kwargs): + raise QueryNotSupported("Bitwise operations not supported") + + +boolean_expression_nodes = { + expressions.EQ: db_func.Equal, + expressions.NEQ: db_func.NotEqual, + expressions.GT: db_func.GreaterThan, + expressions.GTE: db_func.GreaterEqualThan, + expressions.LT: db_func.LowerThan, + expressions.LTE: db_func.LowerEqualThan, + expressions.Is: db_func.Is, + expressions.Like: db_func.Like, + expressions.ILike: lambda a, b: db_func.Like( + functions.Upper(a), functions.Upper(b) + ), +} + + +@register +class BooleanOp(Expression): + def matches(self, expression): + return type(expression) in boolean_expression_nodes + + def to_django(self, expression, recurse, **kwargs): + return ExpressionWrapper( + boolean_expression_nodes[type(expression)]( + recurse(expression.left, **kwargs), + recurse(expression.right, **kwargs), + ), + output_field=BooleanField(), + ) + + +@register +class Between(Expression): + node_class = expressions.Between + + def to_django(self, expression, recurse, **kwargs): + return Q( + ExpressionWrapper( + db_func.GreaterEqualThan( + recurse(expression.this, **kwargs), + recurse(expression.args["low"], **kwargs), + ), + output_field=BooleanField(), + ) + ) & Q( + ExpressionWrapper( + db_func.LowerEqualThan( + recurse(expression.this, **kwargs), + recurse(expression.args["high"], **kwargs), + ), + output_field=BooleanField(), + ) + ) + + +@register +class In(Expression): + node_class = expressions.In + + def to_django(self, expression, recurse, **kwargs): + if expression.args.get("query"): + return ExpressionWrapper( + lookups.In( + recurse(expression.this, **kwargs), + recurse(expression.args["query"], **kwargs), + ), + output_field=BooleanField(), + ) + else: + return ExpressionWrapper( + lookups.In( + recurse(expression.this, **kwargs), + [recurse(e, **kwargs) for e in expression.expressions], + ), + output_field=BooleanField(), + ) + + +@register +class And(Expression): + node_class = expressions.And + + def to_django(self, expression, recurse, **kwargs): + return recurse(expression.left, **kwargs) & recurse(expression.right, **kwargs) + + +@register +class Or(Expression): + node_class = expressions.Or + + def to_django(self, expression, recurse, **kwargs): + return recurse(expression.left, **kwargs) | recurse(expression.right, **kwargs) + + +@register +class Not(Expression): + node_class = expressions.Not + + def to_django(self, expression, recurse, **kwargs): + return ~recurse(expression.this, **kwargs) + + +@register +class Case(Expression): + node_class = expressions.Case + + def to_django(self, expression, recurse, **kwargs): + default = None + whens = [] + if expression.this: + for w in expression.args.get("ifs", []): + whens.append( + models.When( + db_func.Equal( + recurse(expression.this, **kwargs), + recurse(w.this, **kwargs), + ), + then=recurse(w.args["true"], **kwargs), + ) + ) + else: + for w in expression.args.get("ifs", []): + whens.append( + models.When( + recurse(w.this, **kwargs), + then=recurse(w.args["true"], **kwargs), + ) + ) + if expression.args.get("default"): + default = recurse(expression.args["default"], **kwargs) + return db_func.NumericAwareCase(*whens, default=default) + + +@register +class CurrentDate(Expression): + node_class = expressions.CurrentDate + + def to_django(self, expression, recurse, timezone, **kwargs): + return functions.TruncDate(functions.Now(), tzinfo=timezone) + + +@register +class CurrentTime(Expression): + node_class = expressions.CurrentTime + + def to_django(self, expression, recurse, timezone, **kwargs): + return functions.TruncTime(functions.Now(), tzinfo=timezone) + + +@register +class CurrentTimestamp(Expression): + node_class = expressions.CurrentTimestamp + + def to_django(self, expression, recurse, **kwargs): + return functions.Now() + + +@register +class JSONExtract(Expression): + node_class = expressions.JSONExtract + django_transform = KeyTransform + + def to_django(self, expression, recurse, **kwargs): + if isinstance(expression.expression, expressions.JSONPath): + k = recurse(expression.this, **kwargs) + for pathel in expression.expression.expressions: + if isinstance(pathel, expressions.JSONPathRoot): + pass + elif isinstance(pathel, expressions.JSONPathKey): + k = self.django_transform( + pathel.this, + k, + ) + else: + raise QueryNotSupported("Advanced JSON path is not supported") + return k + elif isinstance(expression.expression, expressions.Literal): + return self.django_transform( + expression.expression.this, + recurse(expression.this, **kwargs), + ) + elif isinstance(expression.expression, expressions.Column) or isinstance( + expression.expression, expressions.Identifier + ): + return self.django_transform( + expression.expression.this.this, + recurse(expression.this, **kwargs), + ) + else: + raise QueryNotSupported(f"Unsupported JSON path: {expression.sql()}") + + +@register +class JSONExtractScalar(JSONExtract): + node_class = expressions.JSONExtractScalar + django_transform = KeyTextTransform diff --git a/django_ormql/query.py b/django_ormql/query.py index 0d9ea80..211da6e 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -4,19 +4,11 @@ from django.core.exceptions import FieldError from django.db import models from django.db.models import ( - BooleanField, - ExpressionWrapper, F, OrderBy, - OuterRef, - Q, Value, aggregates, - functions, - lookups, ) -from django.db.models.fields.json import KeyTextTransform, KeyTransform -from django.db.models.functions import Cast from sqlglot import ( Dialect, Generator, @@ -29,8 +21,8 @@ from sqlglot.errors import ANSI_RESET, ANSI_UNDERLINE from . import db_func -from .db_func import NumericAwareCase, _patch_func from .exceptions import QueryError, QueryNotSupported +from .expressions import expression_to_django, expression_to_name, _describe_expression logger = logging.getLogger(__name__) @@ -113,154 +105,6 @@ class Generator(Generator): pass -boolean_expression_nodes = { - expressions.EQ: db_func.Equal, - expressions.NEQ: db_func.NotEqual, - expressions.GT: db_func.GreaterThan, - expressions.GTE: db_func.GreaterEqualThan, - expressions.LT: db_func.LowerThan, - expressions.LTE: db_func.LowerEqualThan, - expressions.Is: db_func.Is, - expressions.Like: db_func.Like, - expressions.ILike: lambda a, b: db_func.Like( - functions.Upper(a), functions.Upper(b) - ), -} - -math_binary_nodes = { - expressions.Mul: db_func.Mul, - expressions.Add: db_func.Add, - expressions.Sub: db_func.Sub, - expressions.Div: db_func.Div, - expressions.Mod: db_func.Mod, -} - -aggregate_nodes = { - expressions.Avg: aggregates.Avg, - expressions.Count: aggregates.Count, - expressions.Max: aggregates.Max, - expressions.Min: aggregates.Min, - expressions.Stddev: aggregates.StdDev, - expressions.Variance: aggregates.Variance, - expressions.Sum: aggregates.Sum, -} - -function_nodes = { - expressions.Coalesce: _patch_func(functions.Coalesce), - expressions.Concat: db_func.PatchedConcatPair, - expressions.Greatest: _patch_func(functions.Greatest), - expressions.Least: _patch_func(functions.Least), - expressions.Abs: _patch_func(functions.Abs), - expressions.Ceil: _patch_func(functions.Ceil), - expressions.Floor: _patch_func(functions.Floor), - expressions.Mod: _patch_func(functions.Mod), - expressions.Left: _patch_func(functions.Left), - expressions.Right: _patch_func(functions.Right), - expressions.Length: _patch_func(functions.Length), - expressions.Lower: _patch_func(functions.Lower), - expressions.Upper: _patch_func(functions.Upper), - expressions.SubstringIndex: _patch_func(functions.StrIndex), -} - -types = { - expressions.DataType.Type.BIGDECIMAL: models.DecimalField( - max_digits=20, decimal_places=2 - ), # TODO variable? - expressions.DataType.Type.DECIMAL: models.DecimalField( - max_digits=20, decimal_places=2 - ), # TODO variable? - expressions.DataType.Type.BIGINT: models.BigIntegerField(), - expressions.DataType.Type.BIGSERIAL: models.BigIntegerField(), - expressions.DataType.Type.INT: models.IntegerField(), - expressions.DataType.Type.BOOLEAN: models.BooleanField(), - expressions.DataType.Type.JSON: models.JSONField(), - expressions.DataType.Type.JSONB: models.JSONField(), - expressions.DataType.Type.DOUBLE: models.FloatField(), - expressions.DataType.Type.FLOAT: models.FloatField(), - expressions.DataType.Type.TEXT: models.TextField(), - expressions.DataType.Type.TIME: models.TimeField(), - expressions.DataType.Type.TIMESTAMPTZ: models.DateTimeField(), - expressions.DataType.Type.DATETIME: models.DateTimeField(), - expressions.DataType.Type.DATE: models.DateField(), -} - - -# Maps Django's Field.get_internal_type() to the ORMQL sql_type vocabulary -# emitted by BaseColumn.sql_type subclasses (see columns.py). Keeping one -# vocabulary means /tables/ introspection and per-query result metadata speak -# the same language. -INTERNAL_TYPE_TO_SQL_TYPE = { - "DateField": "DATE", - "DateTimeField": "DATETIME", - "TimeField": "TIME", - "DurationField": "DURATION", - "IntegerField": "INT", - "BigIntegerField": "INT", - "SmallIntegerField": "INT", - "PositiveIntegerField": "INT", - "PositiveSmallIntegerField": "INT", - "PositiveBigIntegerField": "INT", - "AutoField": "INT", - "BigAutoField": "INT", - "SmallAutoField": "INT", - "DecimalField": "DECIMAL", - "FloatField": "FLOAT", - "BooleanField": "BOOLEAN", - "JSONField": "JSONB", - "CharField": "TEXT", - "TextField": "TEXT", - "EmailField": "TEXT", - "URLField": "TEXT", - "SlugField": "TEXT", - "UUIDField": "TEXT", - "GenericIPAddressField": "TEXT", - "FileField": "TEXT", - "FilePathField": "TEXT", - "ImageField": "TEXT", - "BinaryField": "TEXT", -} - - -def _describe_expression(expr, query=None): - """Extract (sql_type, nullable, hint) from a Django expression's output_field. - Hint is dependent on type and currently only available for truncated datetimes. - - Aggregates and Cast() / typed Func() have `.output_field` directly. Plain - F() references only know their field once resolved against a query; when - `query` is supplied, we resolve on a cloned query (so we don't mutate the - original with extra joins) and read `output_field` off the resolved node. - - Returns ("", None, None) for anything we can't pin down — the frontend treats - that as "unknown, render opaquely" which matches the pre-metadata - behavior. - """ - out = None - try: - out = expr.output_field - except: # noqa - out = None - if out is None and query is not None: - try: - resolved = expr.resolve_expression(query=query.chain(), allow_joins=True) - out = resolved.output_field - except: # noqa - out = None - if out is None: - return "", None - try: - internal = out.get_internal_type() - except: # noqa - return "", None - sql_type = INTERNAL_TYPE_TO_SQL_TYPE.get(internal, "") - nullable = getattr(out, "null", None) - hint = None - - if isinstance(expr, functions.datetime.TruncBase): - hint = {"truncated": expr.kind} - - return sql_type, nullable, hint - - class Result: """Iterable query result carrying per-query column metadata. @@ -286,409 +130,13 @@ def __init__(self, sql, tables, placeholders, timezone, default_limit): self.placeholders = placeholders or {} self.default_limit = default_limit - def _to_column_path(self, expression): - """ - Hack an expression like part1.part2.part3.part4.part5 to [part1, part2, part3, part4, part5] - """ - if isinstance(expression, expressions.Dot): - return [ - *self._to_column_path(expression.this), - expression.expression.this, - ] - elif isinstance(expression, expressions.Column): - return [ - x.this - for x in [ - expression.args.get("catalog"), - expression.args.get("db"), - expression.args.get("table"), - expression.args.get("this"), - ] - if x - ] - else: - raise TypeError("Invalid type") - def _expression_to_django(self, expression, **kwargs): table = kwargs["table"] - aggregate_names = kwargs["aggregate_names"] + # aggregate_names = kwargs["aggregate_names"] parent_table_stack = kwargs.get("parent_table_stack", []) - if isinstance(expression, (expressions.Column, expressions.Dot)): - cp = self._to_column_path(expression) - if len(cp) == 1 and aggregate_names and cp[0] in aggregate_names: - return F(aggregate_names[cp[0]]) - return table.resolve_column_path(cp) - elif ( - isinstance(expression, expressions.Anonymous) - and expression.this.lower() == "outer" - ): + kwargs["timezone"] = self.timezone - def _resolve(e, parent_stack, depth): - if isinstance(e, expressions.Anonymous) and e.this.lower() == "outer": - if not parent_stack: - raise QueryError("OUTER nested too far") - return _resolve(e.expressions[0], parent_stack[:-1], depth + 1) - elif isinstance(e, (expressions.Column, expressions.Dot)): - if not parent_stack: - raise QueryError("OUTER nested too far") - return self._to_column_path(e), parent_stack[-1], depth - else: - raise QueryNotSupported("Invalid argument to OUTER()") - - cp, lookup_table, depth = _resolve( - expression.expressions[0], parent_table_stack, 1 - ) - p = lookup_table.resolve_column_path(cp) - if isinstance(p, F): - p = p.name - else: - raise QueryNotSupported(f"Cannot use '{cp}' in OUTER()") - for i in range(depth): - p = OuterRef(p) - return p - elif isinstance(expression, expressions.Alias): - return self._expression_to_django(expression.this, **kwargs) - elif isinstance(expression, expressions.Literal): - return Value(expression.to_py()) - elif isinstance(expression, expressions.Boolean): - return Value(expression.this) - elif isinstance(expression, expressions.Star): - return "*" - elif isinstance(expression, expressions.Cast): - return Cast( - self._expression_to_django(expression.this, **kwargs), - output_field=types[expression.to.this], - ) - elif isinstance(expression, expressions.Extract): - if isinstance(expression.this, expressions.Var): - lookup_name = expression.this.this.lower() - else: - lookup_name = expression.this.to_py() - if lookup_name not in ( - "year", - "iso_year", - "quarter", - "month", - "day", - "week", - "week_day", - "iso_week_day", - "hour", - "minute", - "second", - ): - raise QueryNotSupported(f"Unsupported extract value '{lookup_name}'") - return functions.Extract( - self._expression_to_django(expression.expression, **kwargs), - lookup_name=lookup_name, - tzinfo=self.timezone, - ) - elif ( - isinstance(expression, expressions.Anonymous) - and expression.this.lower() == "datetrunc" - ): - if len(expression.expressions) != 2: - raise QueryError("Function datetrunc takes exactly two arguments") - try: - lookup_name = expression.expressions[0].to_py() - if lookup_name not in ( - "year", - "quarter", - "month", - "day", - "week", - "hour", - "minute", - "second", - ): - raise QueryNotSupported( - f"Unsupported truncation type '{lookup_name}'" - ) - except ValueError: - raise QueryNotSupported("Unsupported truncation type") - return functions.Trunc( - self._expression_to_django(expression.expressions[1], **kwargs), - lookup_name, - tzinfo=self.timezone, - ) - elif type(expression) in function_nodes: - if expression.args.get("this"): - args = [self._expression_to_django(expression.this, **kwargs)] - else: - args = [] - if expression.args.get("expression"): - args += [self._expression_to_django(expression.expression, **kwargs)] - args += [ - self._expression_to_django(e, **kwargs) for e in expression.expressions - ] - cls = function_nodes[type(expression)] - if (cls.arity and cls.arity != len(args)) or any( - v is not None - and k - not in ( - "this", - "expression", - "expressions", - "ignore_nulls", - "safe", - "coalesce", - ) - for k, v in expression.args.items() - ): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - return cls(*args) - elif isinstance(expression, expressions.Round): - args = [ - self._expression_to_django(expression.this, **kwargs), - ] - if expression.args.get("decimals"): - args.append( - self._expression_to_django(expression.args["decimals"], **kwargs) - ) - if any( - v is not None and k not in ("this", "decimals") - for k, v in expression.args.items() - ): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - return functions.Round(*args) - elif isinstance(expression, expressions.Pad): - args = [ - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.expression, **kwargs), - ] - if expression.args.get("fill_pattern"): - args.append( - self._expression_to_django( - expression.args["fill_pattern"], **kwargs - ) - ) - if any( - v is not None - and k not in ("this", "expression", "fill_pattern", "is_left") - for k, v in expression.args.items() - ): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - if expression.args["is_left"]: - return functions.LPad(*args) - else: - return functions.RPad(*args) - elif isinstance(expression, expressions.StrPosition): - if not expression.args.get("substr"): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - args = [ - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.args["substr"], **kwargs), - ] - if any( - v is not None and k not in ("this", "substr") - for k, v in expression.args.items() - ): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - return functions.StrIndex(*args) - elif isinstance(expression, expressions.Substring): - if not expression.args.get("start"): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - args = [ - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.args["start"], **kwargs), - ] - if expression.args.get("length"): - args.append( - self._expression_to_django(expression.args["length"], **kwargs) - ) - if any( - v is not None and k not in ("this", "start", "length") - for k, v in expression.args.items() - ): - raise QueryNotSupported( - f"Wrong number of arguments for function {expression.sql()}" - ) - return functions.Substr(*args) - elif isinstance(expression, expressions.Replace): - args = [ - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.expression, **kwargs), - ] - if expression.args.get("replacement"): - args.append( - self._expression_to_django(expression.args["replacement"], **kwargs) - ) - return functions.Replace(*args) - elif isinstance(expression, expressions.DPipe): - return functions.Concat( - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.expression, **kwargs), - ) - elif ( - isinstance(expression, expressions.Filter) - and type(expression.this) in aggregate_nodes - ): - if isinstance(expression.this.this, expressions.Distinct): - args = [ - self._expression_to_django(e, **kwargs) - for e in expression.this.this.expressions - ] - distinct = True - else: - args = [self._expression_to_django(expression.this.this, **kwargs)] - distinct = False - if len(args) > 1: - raise QueryNotSupported( - "Multiple arguments to aggregate expression not supported" - ) - return aggregate_nodes[type(expression.this)]( - *args, - distinct=distinct, - filter=self._expression_to_django(expression.expression.this, **kwargs), - ) - elif type(expression) in aggregate_nodes: - if isinstance(expression.this, expressions.Distinct): - args = [ - self._expression_to_django(e, **kwargs) - for e in expression.this.expressions - ] - distinct = True - else: - args = [self._expression_to_django(expression.this, **kwargs)] - distinct = False - if len(args) > 1: - raise QueryNotSupported( - "Multiple arguments to aggregate expression not supported" - ) - return aggregate_nodes[type(expression)](*args, distinct=distinct) - elif type(expression) in math_binary_nodes: - lhs = self._expression_to_django(expression.left, **kwargs) - rhs = self._expression_to_django(expression.right, **kwargs) - return math_binary_nodes[type(expression)]( - lhs, - rhs, - ) - elif isinstance(expression, expressions.Order): - raise QueryNotSupported("ORDER not supported in expression") - elif isinstance(expression, expressions.Null): - # TODO do we need to guess output_field better? - return Value(None, output_field=models.TextField(null=True)) - elif isinstance(expression, (expressions.NullSafeEQ, expressions.NullSafeNEQ)): - raise QueryNotSupported("IS (NOT) DISTINCT not supported") - elif isinstance(expression, expressions.Paren): - return self._expression_to_django(expression.this, **kwargs) - elif isinstance(expression, expressions.Neg): - return -self._expression_to_django(expression.this, **kwargs) - elif isinstance( - expression, - ( - expressions.BitwiseNot, - expressions.BitwiseOr, - expressions.BitwiseXor, - expressions.BitwiseAnd, - expressions.BitwiseCount, - expressions.BitwiseLeftShift, - expressions.BitwiseRightShift, - ), - ): - raise QueryNotSupported("Bitwise operations not supported") - elif type(expression) in boolean_expression_nodes: - return ExpressionWrapper( - boolean_expression_nodes[type(expression)]( - self._expression_to_django(expression.left, **kwargs), - self._expression_to_django(expression.right, **kwargs), - ), - output_field=BooleanField(), - ) - elif isinstance(expression, expressions.Between): - return Q( - ExpressionWrapper( - db_func.GreaterEqualThan( - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.args["low"], **kwargs), - ), - output_field=BooleanField(), - ) - ) & Q( - ExpressionWrapper( - db_func.LowerEqualThan( - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.args["high"], **kwargs), - ), - output_field=BooleanField(), - ) - ) - elif isinstance(expression, expressions.In): - if expression.args.get("query"): - return ExpressionWrapper( - lookups.In( - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(expression.args["query"], **kwargs), - ), - output_field=BooleanField(), - ) - else: - return ExpressionWrapper( - lookups.In( - self._expression_to_django(expression.this, **kwargs), - [ - self._expression_to_django(e, **kwargs) - for e in expression.expressions - ], - ), - output_field=BooleanField(), - ) - elif isinstance(expression, expressions.And): - return self._expression_to_django( - expression.left, **kwargs - ) & self._expression_to_django(expression.right, **kwargs) - elif isinstance(expression, expressions.Or): - return self._expression_to_django( - expression.left, **kwargs - ) | self._expression_to_django(expression.right, **kwargs) - elif isinstance(expression, expressions.Not): - return ~self._expression_to_django(expression.this, **kwargs) - elif isinstance(expression, expressions.Case): - default = None - whens = [] - if expression.this: - for w in expression.args.get("ifs", []): - whens.append( - models.When( - db_func.Equal( - self._expression_to_django(expression.this, **kwargs), - self._expression_to_django(w.this, **kwargs), - ), - then=self._expression_to_django(w.args["true"], **kwargs), - ) - ) - else: - for w in expression.args.get("ifs", []): - whens.append( - models.When( - self._expression_to_django(w.this, **kwargs), - then=self._expression_to_django(w.args["true"], **kwargs), - ) - ) - if expression.args.get("default"): - default = self._expression_to_django( - expression.args["default"], **kwargs - ) - return NumericAwareCase(*whens, default=default) - elif isinstance(expression, expressions.CurrentDate): - return functions.TruncDate(functions.Now(), tzinfo=self.timezone) - elif isinstance(expression, expressions.CurrentTime): - return functions.TruncTime(functions.Now(), tzinfo=self.timezone) - elif isinstance(expression, expressions.CurrentTimestamp): - return functions.Now() - elif isinstance(expression, expressions.Subquery): + if isinstance(expression, expressions.Subquery): if not isinstance(expression.this, expressions.Select): raise QueryNotSupported("Only SELECT subqueries are supported") qs, _, _ = self._select_to_qs( @@ -710,55 +158,10 @@ def _resolve(e, parent_stack, depth): if expression.name not in self.placeholders: raise QueryError(f"Placeholder '{expression.name}' not filled") return Value(self.placeholders[expression.name]) - elif isinstance( - expression, (expressions.JSONExtract, expressions.JSONExtractScalar) - ): - django_transform = ( - KeyTransform - if isinstance(expression, expressions.JSONExtract) - else KeyTextTransform - ) - if isinstance(expression.expression, expressions.JSONPath): - k = self._expression_to_django(expression.this, **kwargs) - for pathel in expression.expression.expressions: - if isinstance(pathel, expressions.JSONPathRoot): - pass - elif isinstance(pathel, expressions.JSONPathKey): - k = django_transform( - pathel.this, - k, - ) - else: - raise QueryNotSupported("Advanced JSON path is not supported") - return k - elif isinstance(expression.expression, expressions.Literal): - return django_transform( - expression.expression.this, - self._expression_to_django(expression.this, **kwargs), - ) - elif isinstance(expression.expression, expressions.Column) or isinstance( - expression.expression, expressions.Identifier - ): - return django_transform( - expression.expression.this.this, - self._expression_to_django(expression.this, **kwargs), - ) - else: - raise QueryNotSupported(f"Unsupported JSON path: {expression.sql()}") else: - raise QueryNotSupported(f"Unsupported expression: {expression.sql()}") - - def _expression_to_name(self, expression): - if isinstance(expression, (expressions.Column, expressions.Dot)): - return ".".join(self._to_column_path(expression)) - elif isinstance(expression, expressions.Literal): - return str(expression.this) - elif isinstance(expression, expressions.Alias): - return expression.output_name - elif type(expression) in aggregate_nodes: - return expression.sql() - else: - return expression.sql() + return expression_to_django( + expression, self._expression_to_django, **kwargs + ) def _where_to_django(self, node, **kwargs): return self._expression_to_django(node, **kwargs) @@ -817,7 +220,7 @@ def _select_to_qs(self, root, parent_table_stack): if isinstance(e, expressions.Star): raise QueryNotSupported("SELECT * is not supported") else: - n = self._expression_to_name(e) + n = expression_to_name(e) while n in values_names or n in aggregations: n += "_" From 50befc8c7a51d96c1179d23f0d46f61d7d940368 Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Tue, 6 Oct 2026 18:10:11 +0200 Subject: [PATCH 2/3] Add type info to expressions --- django_ormql/columns.py | 27 ++- django_ormql/expressions.py | 299 ++++++++++++++++++++++++++++++--- django_ormql/query.py | 34 +--- django_ormql/tables.py | 18 ++ tests/test_expr.py | 286 ++++++++++++++++++------------- tests/test_generated_column.py | 12 ++ tests/test_placeholder.py | 15 ++ tests/test_select_col_from.py | 14 ++ tests/test_subqueries.py | 45 ++++- tests/testapp/ormql.py | 3 +- 10 files changed, 572 insertions(+), 181 deletions(-) diff --git a/django_ormql/columns.py b/django_ormql/columns.py index be0c64a..54016b8 100644 --- a/django_ormql/columns.py +++ b/django_ormql/columns.py @@ -10,10 +10,17 @@ class BaseColumn: def __init__( - self, *, source=None, nullable=None, enum_options=None, model_field=None + self, + *, + source=None, + nullable=None, + sql_type=None, + enum_options=None, + model_field=None, ): self.source = source self._nullable = nullable + self._sql_type = sql_type self.enum_options = enum_options def bind(self, field_name, parent): @@ -29,9 +36,16 @@ def resolve_column_path(self, remaining_path): ) return F(self.source) + def resolve_column_type(self, remaining_path): + if remaining_path: + raise QueryNotSupported( + f"Column '{self.field_name}' is not a related field" + ) + return self.sql_type + @property def sql_type(self): - return "" + return self._sql_type @property def nullable(self): @@ -177,6 +191,15 @@ def resolve_column_path(self, remaining_path): else: return F("__".join([self.source, "pk"])) + def resolve_column_type(self, remaining_path): + if len(remaining_path) > 20: + raise QueryNotSupported("Upper limit of JOINs reached.") + rt = self.related_table(is_related=True) + if remaining_path: + return rt.resolve_column_type(remaining_path) + else: + return "INT" + class GeneratedColumn(BaseColumn): nullable = True diff --git a/django_ormql/expressions.py b/django_ormql/expressions.py index 492123b..b2b34b8 100644 --- a/django_ormql/expressions.py +++ b/django_ormql/expressions.py @@ -1,5 +1,8 @@ -from datetime import datetime, date +import os +from datetime import date, datetime +from decimal import Decimal +from django.conf import settings from django.db import models from django.db.models import ( BooleanField, @@ -33,6 +36,9 @@ def to_django(self, expression, recurse, **kwargs): def to_name(self, expression): return expression.sql() + def to_sql_type(self, expression, recurse, **kwargs): + return self.sql_type + class FuncExpression(Expression): node_class = expressions.Anonymous @@ -64,6 +70,13 @@ def expression_to_name(expression): return expression.sql() +def expression_to_sql_type(expression, **kwargs): + for e in _expressions: + if e.matches(expression): + return e.to_sql_type(expression, expression_to_sql_type, **kwargs) + return None + + types = { expressions.DataType.Type.BIGDECIMAL: models.DecimalField( max_digits=20, decimal_places=2 @@ -148,11 +161,11 @@ def _describe_expression(expr, query=None): except: # noqa out = None if out is None: - return "", None + return "", None, None try: internal = out.get_internal_type() except: # noqa - return "", None + return "", None, None sql_type = INTERNAL_TYPE_TO_SQL_TYPE.get(internal, "") nullable = getattr(out, "null", None) hint = None @@ -162,6 +175,7 @@ def _describe_expression(expr, query=None): return sql_type, nullable, hint + def _to_column_path(expression): """ Hack an expression like part1.part2.part3.part4.part5 to [part1, part2, part3, part4, part5] @@ -200,6 +214,105 @@ def to_django(self, expression, recurse, table, aggregate_names, **kwargs): def to_name(self, expression): return ".".join(_to_column_path(expression)) + def to_sql_type(self, expression, recurse, table, aggregate_names, **kwargs): + cp = _to_column_path(expression) + if len(cp) == 1 and aggregate_names and cp[0] in aggregate_names: + return "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently + return table.resolve_column_type(cp) + + +@register +class Placeholder(Expression): + node_class = expressions.Placeholder + + def to_django(self, expression, recurse, placeholders, **kwargs): + if expression.name == "?": + raise QueryError("Placeholder must be named") + if expression.name not in self.placeholders: + raise QueryError(f"Placeholder '{expression.name}' not filled") + return Value(placeholders[expression.name]) + + def to_name(self, expression): + return expression.name + + def to_sql_type(self, expression, recurse, **kwargs): + placeholders = kwargs.get("placeholders") + if placeholders is None: + return None + if expression.name == "?": + raise QueryError("Placeholder must be named") + if expression.name not in placeholders: + raise QueryError(f"Placeholder '{expression.name}' not filled") + v = placeholders[expression.name] + if isinstance(v, str): + return "TEXT" + elif isinstance(v, int): + return "INT" + elif isinstance(v, float): + return "FLOAT" + elif isinstance(v, datetime): + return "DATETIME" + elif isinstance(v, date): + return "DATE" + elif isinstance(v, bool): + return "BOOLEAN" + elif isinstance(v, Decimal): + return "DECIMAL" + elif isinstance(v, (dict, list)): + return "JSONB" + return None + + +@register +class Subquery(Expression): + node_class = expressions.Subquery + + def to_django(self, expression, recurse, **kwargs): + parent_table_stack = kwargs.get("parent_table_stack", []) + subquery_builder = kwargs["subquery_builder"] + table = kwargs["table"] + if not isinstance(expression.this, expressions.Select): + raise QueryNotSupported("Only SELECT subqueries are supported") + qs, _, _ = subquery_builder( + expression.this, parent_table_stack=parent_table_stack + [table] + ) + return db_func.AutoTypedSubquery( + qs, + ) + + def to_sql_type(self, expression, recurse, **kwargs): + return INTERNAL_TYPE_TO_SQL_TYPE[self.to_django(expression, recurse, **kwargs)._resolve_output_field().get_internal_type()] + + +@register +class Select(Expression): + node_class = expressions.Select + + def to_django(self, expression, recurse, **kwargs): + return Subquery().to_django(expressions.Subquery(this=expression), recurse, **kwargs) + + def to_sql_type(self, expression, recurse, **kwargs): + return Subquery().to_sql_type(expressions.Subquery(this=expression), recurse, **kwargs) + + +@register +class Exists(Expression): + node_class = expressions.Exists + sql_type = "BOOLEAN" + + def to_django(self, expression, recurse, **kwargs): + parent_table_stack = kwargs.get("parent_table_stack", []) + subquery_builder = kwargs["subquery_builder"] + table = kwargs["table"] + if not isinstance(expression.this, expressions.Select): + raise QueryNotSupported("Only SELECT subqueries are supported") + qs, _, _ = subquery_builder( + expression.this, parent_table_stack=parent_table_stack + [table] + ) + return models.Exists( + qs, + ) + @register class Alias(Expression): @@ -211,6 +324,9 @@ def to_django(self, expression, recurse, **kwargs): def to_name(self, expression): return expression.output_name + def to_sql_type(self, expression, recurse, **kwargs): + return recurse(expression.this, **kwargs) + @register class Literal(Expression): @@ -222,6 +338,26 @@ def to_django(self, expression, recurse, **kwargs): def to_name(self, expression): return str(expression.this) + def to_sql_type(self, expression, recurse, **kwargs): + v = expression.to_py() + if isinstance(v, str): + return "TEXT" + elif isinstance(v, int): + return "INT" + elif isinstance(v, float): + return "FLOAT" + elif isinstance(v, datetime): + return "DATETIME" + elif isinstance(v, date): + return "DATE" + elif isinstance(v, bool): + return "BOOLEAN" + elif isinstance(v, Decimal): + return "DECIMAL" + elif isinstance(v, (dict, list)): + return "JSONB" + return None + @register class Boolean(Expression): @@ -251,10 +387,14 @@ def to_django(self, expression, recurse, **kwargs): output_field=types[expression.to.this], ) + def to_sql_type(self, expression, recurse, **kwargs): + return INTERNAL_TYPE_TO_SQL_TYPE[types[expression.to.this].get_internal_type()] + @register class Extract(Expression): node_class = expressions.Extract + sql_type = "INT" def to_django(self, expression, recurse, timezone, **kwargs): if isinstance(expression.this, expressions.Var): @@ -313,6 +453,26 @@ def _resolve(e, parent_stack, depth): p = OuterRef(p) return p + def to_sql_type(self, expression, recurse, **kwargs): + parent_table_stack = kwargs.get("parent_table_stack", []) + + def _resolve(e, parent_stack, depth): + if isinstance(e, expressions.Anonymous) and e.this.lower() == "outer": + if not parent_stack: + raise QueryError("OUTER nested too far") + return _resolve(e.expressions[0], parent_stack[:-1], depth + 1) + elif isinstance(e, (expressions.Column, expressions.Dot)): + if not parent_stack: + raise QueryError("OUTER nested too far") + return _to_column_path(e), parent_stack[-1], depth + else: + raise QueryNotSupported("Invalid argument to OUTER()") + + cp, lookup_table, depth = _resolve( + expression.expressions[0], parent_table_stack, 1 + ) + return lookup_table.resolve_column_type(cp) + @register class Datetrunc(FuncExpression): @@ -342,29 +502,15 @@ def to_django(self, expression, recurse, timezone, **kwargs): tzinfo=timezone, ) - -function_nodes = { - expressions.Coalesce: _patch_func(functions.Coalesce), - expressions.Concat: db_func.PatchedConcatPair, - expressions.Greatest: _patch_func(functions.Greatest), - expressions.Least: _patch_func(functions.Least), - expressions.Abs: _patch_func(functions.Abs), - expressions.Ceil: _patch_func(functions.Ceil), - expressions.Floor: _patch_func(functions.Floor), - expressions.Mod: _patch_func(functions.Mod), - expressions.Left: _patch_func(functions.Left), - expressions.Right: _patch_func(functions.Right), - expressions.Length: _patch_func(functions.Length), - expressions.Lower: _patch_func(functions.Lower), - expressions.Upper: _patch_func(functions.Upper), - expressions.SubstringIndex: _patch_func(functions.StrIndex), -} + def to_sql_type(self, expression, recurse, **kwargs): + if recurse(expression.expressions[1], **kwargs) == "DATE": + return "DATE" + return "DATETIME" -@register -class Function(Expression): +class BaseFunction(Expression): def matches(self, expression): - return type(expression) in function_nodes + return type(expression) in self.function_nodes def to_django(self, expression, recurse, **kwargs): if expression.args.get("this"): @@ -374,7 +520,7 @@ def to_django(self, expression, recurse, **kwargs): if expression.args.get("expression"): args += [recurse(expression.expression, **kwargs)] args += [recurse(e, **kwargs) for e in expression.expressions] - cls = function_nodes[type(expression)] + cls = self.function_nodes[type(expression)] if (cls.arity and cls.arity != len(args)) or any( v is not None and k @@ -394,9 +540,61 @@ def to_django(self, expression, recurse, **kwargs): return cls(*args) +@register +class TextFunction(BaseFunction): + function_nodes = { + expressions.Concat: db_func.PatchedConcatPair, + expressions.Left: _patch_func(functions.Left), + expressions.Right: _patch_func(functions.Right), + expressions.Lower: _patch_func(functions.Lower), + expressions.Upper: _patch_func(functions.Upper), + } + sql_type = "TEXT" + + +@register +class TextToIntFunction(BaseFunction): + function_nodes = { + expressions.Length: _patch_func(functions.Length), + expressions.SubstringIndex: _patch_func(functions.StrIndex), + } + sql_type = "INT" + + +@register +class NumericFunction(BaseFunction): + function_nodes = { + expressions.Greatest: _patch_func(functions.Greatest), + expressions.Least: _patch_func(functions.Least), + expressions.Abs: _patch_func(functions.Abs), + expressions.Ceil: _patch_func(functions.Ceil), + expressions.Floor: _patch_func(functions.Floor), + expressions.Mod: _patch_func(functions.Mod), + } + sql_type = "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently + + +@register +class CoalesceFunction(BaseFunction): + function_nodes = { + expressions.Coalesce: _patch_func(functions.Coalesce), + } + + def to_sql_type(self, expression, recurse, **kwargs): + if expression.args.get("this"): + args = [recurse(expression.this, **kwargs)] + else: + args = [] + if expression.args.get("expression"): + args += [recurse(expression.expression, **kwargs)] + args += [recurse(e, **kwargs) for e in expression.expressions] + return args[0] + + @register class Round(Expression): node_class = expressions.Round + sql_type = "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently def to_django(self, expression, recurse, **kwargs): args = [ @@ -417,6 +615,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Pad(Expression): node_class = expressions.Pad + sql_type = "TEXT" def to_django(self, expression, recurse, **kwargs): args = [ @@ -441,6 +640,7 @@ def to_django(self, expression, recurse, **kwargs): @register class StrPosition(Expression): node_class = expressions.StrPosition + sql_type = "INT" def to_django(self, expression, recurse, **kwargs): if not expression.args.get("substr"): @@ -464,6 +664,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Substring(Expression): node_class = expressions.Substring + sql_type = "TEXT" def to_django(self, expression, recurse, **kwargs): if not expression.args.get("start"): @@ -489,6 +690,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Replace(Expression): node_class = expressions.Replace + sql_type = "TEXT" def to_django(self, expression, recurse, **kwargs): args = [ @@ -503,6 +705,7 @@ def to_django(self, expression, recurse, **kwargs): @register class DPipe(Expression): node_class = expressions.DPipe + sql_type = "TEXT" def to_django(self, expression, recurse, **kwargs): return functions.Concat( @@ -525,6 +728,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Filter(Expression): node_class = expressions.Filter + sql_type = "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently def matches(self, node): return super().matches(node) and type(node.this) in aggregate_nodes @@ -549,6 +753,8 @@ def to_django(self, expression, recurse, **kwargs): @register class Aggregate(Expression): + sql_type = "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently + def matches(self, expression): return type(expression) in aggregate_nodes @@ -578,6 +784,8 @@ def to_django(self, expression, recurse, **kwargs): @register class MathBinary(Expression): + sql_type = "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently + def matches(self, expression): return type(expression) in math_binary_nodes @@ -601,6 +809,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Null(Expression): node_class = expressions.Null + sql_type = "TEXT" def to_django(self, expression, recurse, **kwargs): # TODO do we need to guess output_field better? @@ -630,6 +839,9 @@ class Paren(Expression): def to_django(self, expression, recurse, **kwargs): return recurse(expression.this, **kwargs) + def to_sql_type(self, expression, recurse, **kwargs): + return recurse(expression.this, **kwargs) + @register class Neg(Expression): @@ -638,6 +850,9 @@ class Neg(Expression): def to_django(self, expression, recurse, **kwargs): return -recurse(expression.this, **kwargs) + def to_sql_type(self, expression, recurse, **kwargs): + return recurse(expression.this, **kwargs) + @register class Bitwise(Expression): @@ -676,6 +891,8 @@ def to_django(self, expression, recurse, **kwargs): @register class BooleanOp(Expression): + sql_type = "BOOLEAN" + def matches(self, expression): return type(expression) in boolean_expression_nodes @@ -692,6 +909,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Between(Expression): node_class = expressions.Between + sql_type = "BOOLEAN" def to_django(self, expression, recurse, **kwargs): return Q( @@ -716,6 +934,7 @@ def to_django(self, expression, recurse, **kwargs): @register class In(Expression): node_class = expressions.In + sql_type = "BOOLEAN" def to_django(self, expression, recurse, **kwargs): if expression.args.get("query"): @@ -739,6 +958,7 @@ def to_django(self, expression, recurse, **kwargs): @register class And(Expression): node_class = expressions.And + sql_type = "BOOLEAN" def to_django(self, expression, recurse, **kwargs): return recurse(expression.left, **kwargs) & recurse(expression.right, **kwargs) @@ -747,6 +967,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Or(Expression): node_class = expressions.Or + sql_type = "BOOLEAN" def to_django(self, expression, recurse, **kwargs): return recurse(expression.left, **kwargs) | recurse(expression.right, **kwargs) @@ -755,6 +976,7 @@ def to_django(self, expression, recurse, **kwargs): @register class Not(Expression): node_class = expressions.Not + sql_type = "BOOLEAN" def to_django(self, expression, recurse, **kwargs): return ~recurse(expression.this, **kwargs) @@ -790,10 +1012,21 @@ def to_django(self, expression, recurse, **kwargs): default = recurse(expression.args["default"], **kwargs) return db_func.NumericAwareCase(*whens, default=default) + def to_sql_type(self, expression, recurse, **kwargs): + if expression.this: + for w in expression.args.get("ifs", []): + return recurse(w.args["true"], **kwargs) + else: + for w in expression.args.get("ifs", []): + return recurse(w.args["true"], **kwargs) + if expression.args.get("default"): + return recurse(expression.args["default"], **kwargs) + @register class CurrentDate(Expression): node_class = expressions.CurrentDate + sql_type = "DATE" def to_django(self, expression, recurse, timezone, **kwargs): return functions.TruncDate(functions.Now(), tzinfo=timezone) @@ -802,6 +1035,7 @@ def to_django(self, expression, recurse, timezone, **kwargs): @register class CurrentTime(Expression): node_class = expressions.CurrentTime + sql_type = "TIME" def to_django(self, expression, recurse, timezone, **kwargs): return functions.TruncTime(functions.Now(), tzinfo=timezone) @@ -810,6 +1044,7 @@ def to_django(self, expression, recurse, timezone, **kwargs): @register class CurrentTimestamp(Expression): node_class = expressions.CurrentTimestamp + sql_type = "DATETIME" def to_django(self, expression, recurse, **kwargs): return functions.Now() @@ -819,6 +1054,7 @@ def to_django(self, expression, recurse, **kwargs): class JSONExtract(Expression): node_class = expressions.JSONExtract django_transform = KeyTransform + sql_type = "JSONB" def to_django(self, expression, recurse, **kwargs): if isinstance(expression.expression, expressions.JSONPath): @@ -854,3 +1090,18 @@ def to_django(self, expression, recurse, **kwargs): class JSONExtractScalar(JSONExtract): node_class = expressions.JSONExtractScalar django_transform = KeyTextTransform + sql_type = "TEXT" + + +@register +class TypeInfo(FuncExpression): + func_name = "type_info" + sql_type = "TEXT" + + def to_django(self, expression, recurse, **kwargs): + if not settings.DEBUG and "PYTEST_CURRENT_TEST" not in os.environ: + raise QueryNotSupported(f"TYPE_INFO not supported in production as it is not a stable API") + return Value( + expression_to_sql_type(expression.expressions[0], **kwargs), + output_field=models.TextField(null=True), + ) diff --git a/django_ormql/query.py b/django_ormql/query.py index 211da6e..6ba78ab 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -22,7 +22,7 @@ from . import db_func from .exceptions import QueryError, QueryNotSupported -from .expressions import expression_to_django, expression_to_name, _describe_expression +from .expressions import _describe_expression, expression_to_django, expression_to_name logger = logging.getLogger(__name__) @@ -135,33 +135,11 @@ def _expression_to_django(self, expression, **kwargs): # aggregate_names = kwargs["aggregate_names"] parent_table_stack = kwargs.get("parent_table_stack", []) kwargs["timezone"] = self.timezone - - if isinstance(expression, expressions.Subquery): - if not isinstance(expression.this, expressions.Select): - raise QueryNotSupported("Only SELECT subqueries are supported") - qs, _, _ = self._select_to_qs( - expression.this, parent_table_stack=parent_table_stack + [table] - ) - return db_func.AutoTypedSubquery( - qs, - ) - elif isinstance(expression, expressions.Exists): - if not isinstance(expression.this, expressions.Select): - raise QueryNotSupported("Only SELECT subqueries are supported") - qs, _, _ = self._select_to_qs( - expression.this, parent_table_stack=parent_table_stack + [table] - ) - return models.Exists(qs) - elif isinstance(expression, expressions.Placeholder): - if expression.name == "?": - raise QueryError("Placeholder must be named") - if expression.name not in self.placeholders: - raise QueryError(f"Placeholder '{expression.name}' not filled") - return Value(self.placeholders[expression.name]) - else: - return expression_to_django( - expression, self._expression_to_django, **kwargs - ) + kwargs["placeholders"] = self.placeholders + kwargs["subquery_builder"] = self._select_to_qs + return expression_to_django( + expression, self._expression_to_django, **kwargs + ) def _where_to_django(self, node, **kwargs): return self._expression_to_django(node, **kwargs) diff --git a/django_ormql/tables.py b/django_ormql/tables.py index 25e67c6..b361ef2 100644 --- a/django_ormql/tables.py +++ b/django_ormql/tables.py @@ -257,3 +257,21 @@ def resolve_column_path(self, column_path): column = self.columns[column_name] return column.resolve_column_path(column_path[1:]) + + def resolve_column_type(self, column_path): + for c in column_path: + if "__" in c: + raise QueryError("Cannot use __ in column path") + column_name = column_path[0] + exclude_if_related = getattr(self.Meta, "exclude_if_related", []) + if column_name not in self.columns: + if column_name in exclude_if_related: + raise QueryError( + f"Column '{column_path[0]}' cannot be queried on related tables." + ) + raise QueryError( + f"Column '{column_path[0]}' does not exist in table '{self.Meta.name}'." + ) + + column = self.columns[column_name] + return column.resolve_column_type(column_path[1:]) diff --git a/tests/test_expr.py b/tests/test_expr.py index 078c672..ef94730 100644 --- a/tests/test_expr.py +++ b/tests/test_expr.py @@ -14,101 +14,109 @@ @pytest.mark.django_db @pytest.mark.parametrize( - "expr,result", + "expr,result,type_guess", [ # String quotes - ('"foo"', "foo"), - ("'foo'", "foo"), + ('"foo"', "foo", "TEXT"), + ("'foo'", "foo", "TEXT"), # Number formats - ("10e2", Decimal(1000)), + ("10e2", Decimal(1000), "DECIMAL"), # Binary operators - ("10.5::float + 10::int", Decimal("20.5")), - ("single_price::decimal + 10.5::float + 10::int", Decimal("39.5")), - ("single_price + 10", Decimal("29.00")), - ("single_price + 10.5", Decimal("29.5")), - ("single_price + quantity", Decimal("22.0")), - ("single_price - 10", Decimal("9.00")), - ("single_price - 10.5", Decimal("8.5")), - ("single_price - quantity", Decimal("16.0")), - ("single_price * 10", Decimal("190.00")), - ("single_price * 10.5", Decimal("199.5")), - ("single_price * quantity", Decimal("57.0")), - ("single_price / 10", Decimal("1.90")), - ("ROUND(single_price / 10.5, 2)", Decimal("1.81")), - ("ROUND(single_price / quantity, 2)", Decimal("6.33")), - ("single_price % 3", Decimal("1.00")), - ("single_price % 3.0", Decimal("1.00")), - ("single_price % quantity", Decimal("1.00")), + # Decimals being reported as floats is currently expected, we haven't implemented the logic since we have no need + ("10.5::float + 10::int", Decimal("20.5"), "FLOAT"), + ("single_price::decimal + 10.5::float + 10::int", Decimal("39.5"), "FLOAT"), + ("single_price + 10", Decimal("29.00"), "FLOAT"), + ("single_price + 10.5", Decimal("29.5"), "FLOAT"), + ("single_price + quantity", Decimal("22.0"), "FLOAT"), + ("single_price - 10", Decimal("9.00"), "FLOAT"), + ("single_price - 10.5", Decimal("8.5"), "FLOAT"), + ("single_price - quantity", Decimal("16.0"), "FLOAT"), + ("single_price * 10", Decimal("190.00"), "FLOAT"), + ("single_price * 10.5", Decimal("199.5"), "FLOAT"), + ("single_price * quantity", Decimal("57.0"), "FLOAT"), + ("single_price / 10", Decimal("1.90"), "FLOAT"), + ("ROUND(single_price / 10.5, 2)", Decimal("1.81"), "FLOAT"), + ("ROUND(single_price / quantity, 2)", Decimal("6.33"), "FLOAT"), + ("single_price % 3", Decimal("1.00"), "FLOAT"), + ("single_price % 3.0", Decimal("1.00"), "FLOAT"), + ("single_price % quantity", Decimal("1.00"), "FLOAT"), # Boolean operators - ("single_price > 12", True), - ("single_price < 12", False), - ("(single_price > 12) OR (single_price < 3)", True), - ("(single_price > 12) AND (single_price < 3)", False), - ("single_price IN (19, 20)", True), - ("single_price BETWEEN 19 AND 20", True), - ("TRUE AND FALSE", False), + ("single_price > 12", True, "BOOLEAN"), + ("single_price < 12", False, "BOOLEAN"), + ("(single_price > 12) OR (single_price < 3)", True, "BOOLEAN"), + ("(single_price > 12) AND (single_price < 3)", False, "BOOLEAN"), + ("single_price IN (19, 20)", True, "BOOLEAN"), + ("single_price BETWEEN 19 AND 20", True, "BOOLEAN"), + ("TRUE AND FALSE", False, "BOOLEAN"), # Unary operators - ("+single_price", Decimal(19)), - ("-single_price", Decimal(-19)), + ("+single_price", Decimal(19), "DECIMAL"), + ("-single_price", Decimal(-19), "DECIMAL"), # Combinations - ("-(single_price + 1)", Decimal(-20)), - ("(single_price - 1) * 10", Decimal("180.00")), + ("-(single_price + 1)", Decimal(-20), "FLOAT"), + ("(single_price - 1) * 10", Decimal("180.00"), "FLOAT"), ], ) -def test_simple_math(engine_t1, expr, result): +def test_simple_math(engine_t1, expr, result, type_guess): res = engine_t1.query( f""" - SELECT single_price, quantity, {expr} AS result + SELECT single_price, quantity, {expr} AS result, TYPE_INFO({expr}) AS type_guess FROM orderpositions WHERE quantity = 3 """ ) assert list(res) == [ - {"single_price": Decimal("19.00"), "quantity": 3, "result": result}, + { + "single_price": Decimal("19.00"), + "quantity": 3, + "result": result, + "type_guess": type_guess, + }, ] @pytest.mark.django_db @pytest.mark.parametrize( - "expr,result", + "expr,result,type_guess", [ - ("CAST(single_price AS TEXT)", re.compile(r"19(\.00)?")), - ("single_price::TEXT", re.compile(r"19(\.00)?")), - ("CAST(single_price AS INT)", 19), - ("single_price::INT", 19), - ("CAST(single_price AS BIGINT)", 19), - ("single_price::BIGINT", 19), - ("CAST(single_price AS DECIMAL)", Decimal("19.00")), - ("single_price::DECIMAL", Decimal("19.00")), - ("CAST(single_price AS FLOAT)", 19.00), - ("single_price::FLOAT", 19.00), - ("CAST(single_price AS DOUBLE)", 19.00), - ("single_price::DOUBLE", 19.00), - ("CAST(single_price > 0 AS BOOL)", True), - ("(single_price > 0)::BOOL", True), - ("CAST(single_price > 0 AS BOOLEAN)", True), - ("(single_price > 0)::BOOLEAN", True), + ("CAST(single_price AS TEXT)", re.compile(r"19(\.00)?"), "TEXT"), + ("single_price::TEXT", re.compile(r"19(\.00)?"), "TEXT"), + ("CAST(single_price AS INT)", 19, "INT"), + ("single_price::INT", 19, "INT"), + ("CAST(single_price AS BIGINT)", 19, "INT"), + ("single_price::BIGINT", 19, "INT"), + ("CAST(single_price AS DECIMAL)", Decimal("19.00"), "DECIMAL"), + ("single_price::DECIMAL", Decimal("19.00"), "DECIMAL"), + ("CAST(single_price AS FLOAT)", 19.00, "FLOAT"), + ("single_price::FLOAT", 19.00, "FLOAT"), + ("CAST(single_price AS DOUBLE)", 19.00, "FLOAT"), + ("single_price::DOUBLE", 19.00, "FLOAT"), + ("CAST(single_price > 0 AS BOOL)", True, "BOOLEAN"), + ("(single_price > 0)::BOOL", True, "BOOLEAN"), + ("CAST(single_price > 0 AS BOOLEAN)", True, "BOOLEAN"), + ("(single_price > 0)::BOOLEAN", True, "BOOLEAN"), ( "CAST(order.created AS DATETIME)", datetime.datetime(2024, 12, 14, 2, 13, 14, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "order.created::DATETIME", datetime.datetime(2024, 12, 14, 2, 13, 14, tzinfo=datetime.timezone.utc), + "DATETIME", ), - ("CAST(order.created AS DATE)", datetime.date(2024, 12, 14)), - ("order.created::DATE", datetime.date(2024, 12, 14)), - ("CAST(order.created AS TIME)", datetime.time(2, 13, 14)), - ("order.created::TIME", datetime.time(2, 13, 14)), + ("CAST(order.created AS DATE)", datetime.date(2024, 12, 14), "DATE"), + ("order.created::DATE", datetime.date(2024, 12, 14), "DATE"), + ("CAST(order.created AS TIME)", datetime.time(2, 13, 14), "TIME"), + ("order.created::TIME", datetime.time(2, 13, 14), "TIME"), ], ) -def test_cast(engine_t1, expr, result): +def test_cast(engine_t1, expr, result, type_guess): if isinstance(result, bool) and "sqlite" in settings.DATABASES["default"]["ENGINE"]: pytest.skip("Not supported on SQLite") res = engine_t1.query( f""" - SELECT single_price, quantity, {expr} AS result + SELECT single_price, quantity, {expr} AS result, TYPE_INFO({expr}) AS type_guess FROM orderpositions WHERE quantity = 3 """ @@ -117,7 +125,12 @@ def test_cast(engine_t1, expr, result): assert result.match(list(res)[0]["result"]) else: assert list(res) == [ - {"single_price": Decimal("19.00"), "quantity": 3, "result": result}, + { + "single_price": Decimal("19.00"), + "quantity": 3, + "result": result, + "type_guess": type_guess, + }, ] @@ -180,9 +193,9 @@ def test_case_base_when_no_else(engine_t1): res = engine_t1.query( """ SELECT CASE order.status - WHEN "canceled" - THEN single_price * quantity - END AS revenue + WHEN "canceled" + THEN single_price * quantity + END AS revenue FROM orderpositions """ ) @@ -195,120 +208,150 @@ def test_case_base_when_no_else(engine_t1): ] +@pytest.mark.django_db +def test_case_type_guess(engine_t1): + res = engine_t1.query( + """ + SELECT TYPE_INFO(CASE order.status + WHEN "canceled" + THEN single_price * quantity + END) AS type_info + FROM orderpositions + """ + ) + assert list(res)[0] == {"type_info": "FLOAT"} + + @pytest.mark.django_db @pytest.mark.parametrize( - "expr,result", + "expr,result,type_guess", [ # Number functions - ("GREATEST(4, 3, 5)", 5), - ("GREATEST(4, 3.4, 5)", 5), - ("GREATEST(single_price, tax_rate)", 19), - ("LEAST(4, 3, 5)", 3), - ("LEAST(4.5, 3, 5)", 3), - ("LEAST(single_price, tax_rate)", 19), - ("ABS(single_price)", Decimal("19.00")), - ("ABS(-3)", 3), - ("ABS(-3.2)", Decimal("3.2")), - ("CEIL(single_price / 2)", 10), - ("FLOOR(single_price / 2)", 9), - ("ROUND(single_price / 2)", Decimal("10.00")), - ("ROUND(single_price / 2, 2)", Decimal("9.50")), - ("MOD(single_price, 3)", 1), + ("GREATEST(4, 3, 5)", 5, "FLOAT"), + ("GREATEST(4, 3.4, 5)", 5, "FLOAT"), + ("GREATEST(single_price, tax_rate)", 19, "FLOAT"), + ("LEAST(4, 3, 5)", 3, "FLOAT"), + ("LEAST(4.5, 3, 5)", 3, "FLOAT"), + ("LEAST(single_price, tax_rate)", 19, "FLOAT"), + ("ABS(single_price)", Decimal("19.00"), "FLOAT"), + ("ABS(-3)", 3, "FLOAT"), + ("ABS(-3.2)", Decimal("3.2"), "FLOAT"), + ("CEIL(single_price / 2)", 10, "FLOAT"), + ("FLOOR(single_price / 2)", 9, "FLOAT"), + ("ROUND(single_price / 2)", Decimal("10.00"), "FLOAT"), + ("ROUND(single_price / 2, 2)", Decimal("9.50"), "FLOAT"), + ("MOD(single_price, 3)", 1, "FLOAT"), # Date functions - ("EXTRACT('year' FROM order.created)", 2024), - ("EXTRACT(YEAR FROM order.created)", 2024), - ("EXTRACT('iso_year' FROM order.created)", 2024), - ("EXTRACT(ISO_YEAR FROM order.created)", 2024), - ("EXTRACT('quarter' FROM order.created)", 4), - ("EXTRACT(QUARTER FROM order.created)", 4), - ("EXTRACT('month' FROM order.created)", 12), - ("EXTRACT(MONTH FROM order.created)", 12), - ("EXTRACT('day' FROM order.created)", 14), - ("EXTRACT(DAY FROM order.created)", 14), - ("EXTRACT('week' FROM order.created)", 50), - ("EXTRACT(WEEK FROM order.created)", 50), - ("EXTRACT('week_day' FROM order.created)", 7), - ("EXTRACT(WEEK_DAY FROM order.created)", 7), - ("EXTRACT('iso_week_day' FROM order.created)", 6), - ("EXTRACT(ISO_WEEK_DAY FROM order.created)", 6), - ("EXTRACT('hour' FROM order.created)", 2), - ("EXTRACT(HOUR FROM order.created)", 2), - ("EXTRACT('minute' FROM order.created)", 13), - ("EXTRACT(MINUTE FROM order.created)", 13), - ("EXTRACT('second' FROM order.created)", 14), - ("EXTRACT(SECOND FROM order.created)", 14), + ("EXTRACT('year' FROM order.created)", 2024, "INT"), + ("EXTRACT(YEAR FROM order.created)", 2024, "INT"), + ("EXTRACT('iso_year' FROM order.created)", 2024, "INT"), + ("EXTRACT(ISO_YEAR FROM order.created)", 2024, "INT"), + ("EXTRACT('quarter' FROM order.created)", 4, "INT"), + ("EXTRACT(QUARTER FROM order.created)", 4, "INT"), + ("EXTRACT('month' FROM order.created)", 12, "INT"), + ("EXTRACT(MONTH FROM order.created)", 12, "INT"), + ("EXTRACT('day' FROM order.created)", 14, "INT"), + ("EXTRACT(DAY FROM order.created)", 14, "INT"), + ("EXTRACT('week' FROM order.created)", 50, "INT"), + ("EXTRACT(WEEK FROM order.created)", 50, "INT"), + ("EXTRACT('week_day' FROM order.created)", 7, "INT"), + ("EXTRACT(WEEK_DAY FROM order.created)", 7, "INT"), + ("EXTRACT('iso_week_day' FROM order.created)", 6, "INT"), + ("EXTRACT(ISO_WEEK_DAY FROM order.created)", 6, "INT"), + ("EXTRACT('hour' FROM order.created)", 2, "INT"), + ("EXTRACT(HOUR FROM order.created)", 2, "INT"), + ("EXTRACT('minute' FROM order.created)", 13, "INT"), + ("EXTRACT(MINUTE FROM order.created)", 13, "INT"), + ("EXTRACT('second' FROM order.created)", 14, "INT"), + ("EXTRACT(SECOND FROM order.created)", 14, "INT"), ( "DATETRUNC('year', order.created)", datetime.datetime(2024, 1, 1, 0, 0, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('quarter', order.created)", datetime.datetime(2024, 10, 1, 0, 0, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('month', order.created)", datetime.datetime(2024, 12, 1, 0, 0, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('day', order.created)", datetime.datetime(2024, 12, 14, 0, 0, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('week', order.created)", datetime.datetime(2024, 12, 9, 0, 0, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('hour', order.created)", datetime.datetime(2024, 12, 14, 2, 0, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('minute', order.created)", datetime.datetime(2024, 12, 14, 2, 13, 0, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), ( "DATETRUNC('second', order.created)", datetime.datetime(2024, 12, 14, 2, 13, 14, 0, tzinfo=datetime.timezone.utc), + "DATETIME", ), # String functions ( "CONCAT(product.title, ' for ', order.customer.name)", "Lord of the rings DVD for CA", + "TEXT", ), ( "CONCAT(order.status, ' / ', order.comment)", # combine CharField and TextField "paid / No comment", + "TEXT", ), ( "product.title || ' for ' || order.customer.name", "Lord of the rings DVD for CA", + "TEXT", ), - ("LEFT(product.title, 3)", "Lor"), - ("RIGHT(product.title, 3)", "DVD"), - ("LENGTH(product.title)", 21), - ("LOWER(product.title)", "lord of the rings dvd"), - ("upper(product.title)", "LORD OF THE RINGS DVD"), - ("LPAD(UPPER(product.title), 23)", " LORD OF THE RINGS DVD"), - ("LPAD(UPPER(product.title), 23, '_')", "__LORD OF THE RINGS DVD"), - ("RPAD(UPPER(product.title), 23)", "LORD OF THE RINGS DVD "), - ("RPAD(UPPER(product.title), 23, '_')", "LORD OF THE RINGS DVD__"), - ("REPLACE(product.title, ' ')", "LordoftheringsDVD"), - ("REPLACE(product.title, ' ', '_')", "Lord_of_the_rings_DVD"), - ("INSTR(product.title, 'of')", 6), - ("SUBSTRING(product.title, 6)", "of the rings DVD"), - ("SUBSTRING(product.title, INSTR(product.title, 'DVD'))", "DVD"), - ("SUBSTRING(product.title, 6, 2)", "of"), + ("LEFT(product.title, 3)", "Lor", "TEXT"), + ("RIGHT(product.title, 3)", "DVD", "TEXT"), + ("LENGTH(product.title)", 21, "INT"), + ("LOWER(product.title)", "lord of the rings dvd", "TEXT"), + ("upper(product.title)", "LORD OF THE RINGS DVD", "TEXT"), + ("LPAD(UPPER(product.title), 23)", " LORD OF THE RINGS DVD", "TEXT"), + ("LPAD(UPPER(product.title), 23, '_')", "__LORD OF THE RINGS DVD", "TEXT"), + ("RPAD(UPPER(product.title), 23)", "LORD OF THE RINGS DVD ", "TEXT"), + ("RPAD(UPPER(product.title), 23, '_')", "LORD OF THE RINGS DVD__", "TEXT"), + ("REPLACE(product.title, ' ')", "LordoftheringsDVD", "TEXT"), + ("REPLACE(product.title, ' ', '_')", "Lord_of_the_rings_DVD", "TEXT"), + ("INSTR(product.title, 'of')", 6, "INT"), + ("SUBSTRING(product.title, 6)", "of the rings DVD", "TEXT"), + ("SUBSTRING(product.title, INSTR(product.title, 'DVD'))", "DVD", "TEXT"), + ("SUBSTRING(product.title, 6, 2)", "of", "TEXT"), ], ) -def test_functions(engine_t1, expr, result): +def test_functions(engine_t1, expr, result, type_guess): res = engine_t1.query( f""" - SELECT single_price, quantity, {expr} AS result + SELECT single_price, quantity, {expr} AS result, TYPE_INFO({expr}) AS type_guess FROM orderpositions WHERE quantity = 3 """ ) assert list(res) == [ - {"single_price": Decimal("19.00"), "quantity": 3, "result": result}, + { + "single_price": Decimal("19.00"), + "quantity": 3, + "result": result, + "type_guess": type_guess, + }, ] @@ -365,13 +408,14 @@ def test_current_date(engine_t1): res = list( engine_t1.query( """ - SELECT CURRENT_DATE AS d + SELECT CURRENT_DATE AS d, TYPE_INFO(CURRENT_DATE) AS type_guess FROM orderpositions WHERE quantity = 3 """ ) ) assert type(res[0]["d"]) is datetime.date + assert res[0]["type_guess"] == "DATE" assert abs(now().date() - res[0]["d"]) < datetime.timedelta(days=1) @@ -380,13 +424,14 @@ def test_current_datetime(engine_t1): res = list( engine_t1.query( """ - SELECT CURRENT_TIMESTAMP AS d + SELECT CURRENT_TIMESTAMP AS d, TYPE_INFO(CURRENT_TIMESTAMP) AS type_guess FROM orderpositions WHERE quantity = 3 """ ) ) assert type(res[0]["d"]) is datetime.datetime + assert res[0]["type_guess"] == "DATETIME" assert abs(now() - res[0]["d"]) < datetime.timedelta(minutes=1) @@ -395,13 +440,14 @@ def test_current_time(engine_t1): res = list( engine_t1.query( """ - SELECT CURRENT_TIME AS d + SELECT CURRENT_TIME AS d, TYPE_INFO(CURRENT_TIME) AS type_guess FROM orderpositions WHERE quantity = 3 """ ) ) assert type(res[0]["d"]) is datetime.time + assert res[0]["type_guess"] == "TIME" assert now().astimezone(datetime.timezone.utc).time().minute == res[0]["d"].minute diff --git a/tests/test_generated_column.py b/tests/test_generated_column.py index 84686a3..d022ad1 100644 --- a/tests/test_generated_column.py +++ b/tests/test_generated_column.py @@ -16,6 +16,18 @@ def test_select_same_table(engine_t1): ] +@pytest.mark.django_db +def test_select_type_info(engine_t1): + res = engine_t1.query( + """ + SELECT type_info(validity) AS type_info + FROM orders + LIMIT 1 + """ + ) + assert list(res) == [{"type_info": "TEXT"}] + + @pytest.mark.django_db def test_select_joined_table(engine_t1): res = engine_t1.query( diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py index 06d19b7..f7c2114 100644 --- a/tests/test_placeholder.py +++ b/tests/test_placeholder.py @@ -35,3 +35,18 @@ def test_placeholder(engine_t1): ) ) assert res == [{"c": 1}] + + +@pytest.mark.django_db +def test_type_info(engine_t1): + res = list( + engine_t1.query( + """ + SELECT TYPE_INFO(:var) as c + FROM orderpositions + LIMIT 1 + """, + placeholders={"var": 3}, + ) + ) + assert res == [{"c": "INT"}] diff --git a/tests/test_select_col_from.py b/tests/test_select_col_from.py index b37709c..d9705cf 100644 --- a/tests/test_select_col_from.py +++ b/tests/test_select_col_from.py @@ -81,6 +81,20 @@ def test_auto_join_two_levels(engine_t1): ] +@pytest.mark.django_db +def test_auto_join_two_levels_type_info(engine_t1): + res = engine_t1.query( + """ + SELECT TYPE_INFO(order.customer.name) AS type_info + FROM orderpositions + LIMIT 1 + """ + ) + assert list(res) == [ + {"type_info": "TEXT"}, + ] + + @pytest.mark.django_db def test_quote_identifier(engine_t1): res = engine_t1.query( diff --git a/tests/test_subqueries.py b/tests/test_subqueries.py index 12d483f..82dfd98 100644 --- a/tests/test_subqueries.py +++ b/tests/test_subqueries.py @@ -23,13 +23,14 @@ def test_unrelated_subquery(engine_t1): def test_related_subquery(engine_t1): res = engine_t1.query( """ - SELECT (SELECT name FROM customers WHERE id = OUTER(order.customer)) AS result + SELECT (SELECT name FROM customers WHERE id = OUTER(order.customer)) AS result, + TYPE_INFO(SELECT name FROM customers WHERE id = OUTER(order.customer)) AS type_info FROM orderpositions WHERE quantity = 3 """ ) assert list(res) == [ - {"result": "CA"}, + {"result": "CA", "type_info": "TEXT"}, ] @@ -69,20 +70,40 @@ def test_nested_outer_ref_in_subquery(engine_t1): ] +@pytest.mark.django_db +def test_outer_ref_type_resolution(engine_t1): + res = engine_t1.query( + """ + SELECT ( + SELECT ( + SELECT TYPE_INFO(OUTER(OUTER(order.customer.name))) FROM customers WHERE id = OUTER(OUTER(order.customer)) + ) FROM orders WHERE id = OUTER(order) + ) AS result + FROM orderpositions + WHERE quantity = 3 + """ + ) + assert list(res) == [ + {"result": "TEXT"}, + ] + + @pytest.mark.django_db def test_aggregate_in_subquery_with_outerref(engine_t1): res = engine_t1.query( """ SELECT title, ( SELECT COUNT(*) FROM orderpositions WHERE product = OUTER(id) - ) AS result + ) AS result, TYPE_INFO( + SELECT COUNT(*) FROM orderpositions WHERE product = OUTER(id) + ) AS type_info FROM products """ ) assert list(res) == [ - {"title": "Lord of the rings", "result": 2}, - {"title": "SQL for Dummies", "result": 1}, - {"title": "Lord of the rings DVD", "result": 2}, + {"title": "Lord of the rings", "result": 2, "type_info": "INT"}, + {"title": "SQL for Dummies", "result": 1, "type_info": "INT"}, + {"title": "Lord of the rings DVD", "result": 2, "type_info": "INT"}, ] @@ -141,6 +162,18 @@ def test_exists_subquery(engine_t1): assert list(res) == [ {"title": "Lord of the rings"}, ] + res = engine_t1.query( + """ + SELECT + EXISTS(SELECT 1 FROM orderpositions WHERE product = OUTER(id) AND order.status = "paid") exists, + TYPE_INFO(EXISTS(SELECT 1 FROM orderpositions WHERE product = OUTER(id) AND order.status = "paid")) type_info + FROM products + LIMIT 1 + """ + ) + assert list(res) == [ + {"exists": False, "type_info": "BOOLEAN"}, + ] @pytest.mark.django_db diff --git a/tests/testapp/ormql.py b/tests/testapp/ormql.py index 74083a8..f02f55c 100644 --- a/tests/testapp/ormql.py +++ b/tests/testapp/ormql.py @@ -80,7 +80,8 @@ class OrderTable(ModelTable): Case( When(status__in=("new", "paid"), then=Value("valid")), default=Value("invalid"), - ) + ), + sql_type="TEXT", ) email = GeneratedColumn(F("customer__email")) email_upper = GeneratedColumn(Upper(F("customer__email"))) From 9fc75957f215cccf03b0fff1c2683bf89e863576 Mon Sep 17 00:00:00 2001 From: Raphael Michel Date: Tue, 6 Oct 2026 20:26:10 +0200 Subject: [PATCH 3/3] Support DATETRUNC and EXTRACT on DATE and TIME types --- django_ormql/expressions.py | 73 +++++++++++++++++++---- django_ormql/query.py | 10 ++-- tests/test_expr.py | 113 ++++++++++++++++++++++++++++++++++-- tests/testapp/factories.py | 1 + tests/testapp/models.py | 1 + tests/testapp/ormql.py | 1 + 6 files changed, 175 insertions(+), 24 deletions(-) diff --git a/django_ormql/expressions.py b/django_ormql/expressions.py index b2b34b8..e585c29 100644 --- a/django_ormql/expressions.py +++ b/django_ormql/expressions.py @@ -205,7 +205,9 @@ class Column(Expression): def matches(self, expression): return isinstance(expression, (expressions.Column, expressions.Dot)) - def to_django(self, expression, recurse, table, aggregate_names, **kwargs): + def to_django(self, expression, recurse, **kwargs): + table = kwargs["table"] + aggregate_names = kwargs["aggregate_names"] cp = _to_column_path(expression) if len(cp) == 1 and aggregate_names and cp[0] in aggregate_names: return F(aggregate_names[cp[0]]) @@ -214,7 +216,9 @@ def to_django(self, expression, recurse, table, aggregate_names, **kwargs): def to_name(self, expression): return ".".join(_to_column_path(expression)) - def to_sql_type(self, expression, recurse, table, aggregate_names, **kwargs): + def to_sql_type(self, expression, recurse, **kwargs): + table = kwargs["table"] + aggregate_names = kwargs["aggregate_names"] cp = _to_column_path(expression) if len(cp) == 1 and aggregate_names and cp[0] in aggregate_names: return "FLOAT" # we're type-guessing only for now, we don't care if INT or FLOAT or DECIMAL currently @@ -225,10 +229,11 @@ def to_sql_type(self, expression, recurse, table, aggregate_names, **kwargs): class Placeholder(Expression): node_class = expressions.Placeholder - def to_django(self, expression, recurse, placeholders, **kwargs): + def to_django(self, expression, recurse, **kwargs): + placeholders = kwargs["placeholders"] if expression.name == "?": raise QueryError("Placeholder must be named") - if expression.name not in self.placeholders: + if expression.name not in placeholders: raise QueryError(f"Placeholder '{expression.name}' not filled") return Value(placeholders[expression.name]) @@ -281,7 +286,11 @@ def to_django(self, expression, recurse, **kwargs): ) def to_sql_type(self, expression, recurse, **kwargs): - return INTERNAL_TYPE_TO_SQL_TYPE[self.to_django(expression, recurse, **kwargs)._resolve_output_field().get_internal_type()] + return INTERNAL_TYPE_TO_SQL_TYPE[ + self.to_django(expression, recurse, **kwargs) + ._resolve_output_field() + .get_internal_type() + ] @register @@ -289,10 +298,14 @@ class Select(Expression): node_class = expressions.Select def to_django(self, expression, recurse, **kwargs): - return Subquery().to_django(expressions.Subquery(this=expression), recurse, **kwargs) + return Subquery().to_django( + expressions.Subquery(this=expression), recurse, **kwargs + ) def to_sql_type(self, expression, recurse, **kwargs): - return Subquery().to_sql_type(expressions.Subquery(this=expression), recurse, **kwargs) + return Subquery().to_sql_type( + expressions.Subquery(this=expression), recurse, **kwargs + ) @register @@ -396,7 +409,8 @@ class Extract(Expression): node_class = expressions.Extract sql_type = "INT" - def to_django(self, expression, recurse, timezone, **kwargs): + def to_django(self, expression, recurse, **kwargs): + timezone = kwargs["timezone"] if isinstance(expression.this, expressions.Var): lookup_name = expression.this.this.lower() else: @@ -415,6 +429,20 @@ def to_django(self, expression, recurse, timezone, **kwargs): "second", ): raise QueryNotSupported(f"Unsupported extract value '{lookup_name}'") + if expression_to_sql_type(expression.expression, **kwargs) == "DATE": + # tzinfo is only supported for datetimes, not dates + timezone = None + if lookup_name in ("hour", "minute", "second"): + raise QueryNotSupported( + f"Unsupported extract value '{lookup_name}' for DATE" + ) + elif expression_to_sql_type(expression.expression, **kwargs) == "TIME": + # tzinfo is only supported for datetimes, not dates + timezone = None + if lookup_name not in ("hour", "minute", "second"): + raise QueryNotSupported( + f"Unsupported extract value '{lookup_name}' for TIME" + ) return functions.Extract( recurse(expression.expression, **kwargs), lookup_name=lookup_name, @@ -478,7 +506,8 @@ def _resolve(e, parent_stack, depth): class Datetrunc(FuncExpression): func_name = "datetrunc" - def to_django(self, expression, recurse, timezone, **kwargs): + def to_django(self, expression, recurse, **kwargs): + timezone = kwargs["timezone"] if len(expression.expressions) != 2: raise QueryError("Function datetrunc takes exactly two arguments") try: @@ -496,6 +525,20 @@ def to_django(self, expression, recurse, timezone, **kwargs): raise QueryNotSupported(f"Unsupported truncation type '{lookup_name}'") except ValueError: raise QueryNotSupported("Unsupported truncation type") + if expression_to_sql_type(expression.expressions[1], **kwargs) == "DATE": + # tzinfo is only supported for datetimes, not dates + timezone = None + if lookup_name in ("hour", "minute", "second"): + raise QueryNotSupported( + f"Unsupported truncation type '{lookup_name}' for DATE" + ) + elif expression_to_sql_type(expression.expressions[1], **kwargs) == "TIME": + # tzinfo is only supported for datetimes, not times + timezone = None + if lookup_name not in ("hour", "minute", "second"): + raise QueryNotSupported( + f"Unsupported truncation type '{lookup_name}' for TIME" + ) return functions.Trunc( recurse(expression.expressions[1], **kwargs), lookup_name, @@ -505,6 +548,8 @@ def to_django(self, expression, recurse, timezone, **kwargs): def to_sql_type(self, expression, recurse, **kwargs): if recurse(expression.expressions[1], **kwargs) == "DATE": return "DATE" + if recurse(expression.expressions[1], **kwargs) == "TIME": + return "TIME" return "DATETIME" @@ -1028,7 +1073,8 @@ class CurrentDate(Expression): node_class = expressions.CurrentDate sql_type = "DATE" - def to_django(self, expression, recurse, timezone, **kwargs): + def to_django(self, expression, recurse, **kwargs): + timezone = kwargs["timezone"] return functions.TruncDate(functions.Now(), tzinfo=timezone) @@ -1037,7 +1083,8 @@ class CurrentTime(Expression): node_class = expressions.CurrentTime sql_type = "TIME" - def to_django(self, expression, recurse, timezone, **kwargs): + def to_django(self, expression, recurse, **kwargs): + timezone = kwargs["timezone"] return functions.TruncTime(functions.Now(), tzinfo=timezone) @@ -1100,7 +1147,9 @@ class TypeInfo(FuncExpression): def to_django(self, expression, recurse, **kwargs): if not settings.DEBUG and "PYTEST_CURRENT_TEST" not in os.environ: - raise QueryNotSupported(f"TYPE_INFO not supported in production as it is not a stable API") + raise QueryNotSupported( + "TYPE_INFO not supported in production as it is not a stable API" + ) return Value( expression_to_sql_type(expression.expressions[0], **kwargs), output_field=models.TextField(null=True), diff --git a/django_ormql/query.py b/django_ormql/query.py index 6ba78ab..5152b3b 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -20,7 +20,6 @@ ) from sqlglot.errors import ANSI_RESET, ANSI_UNDERLINE -from . import db_func from .exceptions import QueryError, QueryNotSupported from .expressions import _describe_expression, expression_to_django, expression_to_name @@ -131,15 +130,14 @@ def __init__(self, sql, tables, placeholders, timezone, default_limit): self.default_limit = default_limit def _expression_to_django(self, expression, **kwargs): - table = kwargs["table"] + # expected kwargs depending on context: + # table = kwargs["table"] # aggregate_names = kwargs["aggregate_names"] - parent_table_stack = kwargs.get("parent_table_stack", []) + # parent_table_stack = kwargs.get("parent_table_stack", []) kwargs["timezone"] = self.timezone kwargs["placeholders"] = self.placeholders kwargs["subquery_builder"] = self._select_to_qs - return expression_to_django( - expression, self._expression_to_django, **kwargs - ) + return expression_to_django(expression, self._expression_to_django, **kwargs) def _where_to_django(self, node, **kwargs): return self._expression_to_django(node, **kwargs) diff --git a/tests/test_expr.py b/tests/test_expr.py index ef94730..d2dd086 100644 --- a/tests/test_expr.py +++ b/tests/test_expr.py @@ -7,7 +7,7 @@ from django.conf import settings from django.utils.timezone import now -from django_ormql.exceptions import QueryError +from django_ormql.exceptions import QueryError, QueryNotSupported tz_ny = zoneinfo.ZoneInfo("America/New_York") @@ -140,7 +140,7 @@ def test_cast_fail(engine_t1): list( engine_t1.query( """ - SELECT EXTRACT("day", "2025-02-01"::DATE) FROM orderpositions + SELECT EXTRACT("day", "123"::INT) FROM orderpositions """ ) ) @@ -241,7 +241,7 @@ def test_case_type_guess(engine_t1): ("ROUND(single_price / 2)", Decimal("10.00"), "FLOAT"), ("ROUND(single_price / 2, 2)", Decimal("9.50"), "FLOAT"), ("MOD(single_price, 3)", 1, "FLOAT"), - # Date functions + # Datetime functions ("EXTRACT('year' FROM order.created)", 2024, "INT"), ("EXTRACT(YEAR FROM order.created)", 2024, "INT"), ("EXTRACT('iso_year' FROM order.created)", 2024, "INT"), @@ -304,6 +304,65 @@ def test_case_type_guess(engine_t1): datetime.datetime(2024, 12, 14, 2, 13, 14, 0, tzinfo=datetime.timezone.utc), "DATETIME", ), + # Date functions + ("EXTRACT('year' FROM product.publication_date)", 2026, "INT"), + ("EXTRACT(YEAR FROM product.publication_date)", 2026, "INT"), + ("EXTRACT('iso_year' FROM product.publication_date)", 2026, "INT"), + ("EXTRACT(ISO_YEAR FROM product.publication_date)", 2026, "INT"), + ("EXTRACT('quarter' FROM product.publication_date)", 1, "INT"), + ("EXTRACT(QUARTER FROM product.publication_date)", 1, "INT"), + ("EXTRACT('month' FROM product.publication_date)", 2, "INT"), + ("EXTRACT(MONTH FROM product.publication_date)", 2, "INT"), + ("EXTRACT('day' FROM product.publication_date)", 20, "INT"), + ("EXTRACT(DAY FROM product.publication_date)", 20, "INT"), + ("EXTRACT('week' FROM product.publication_date)", 8, "INT"), + ("EXTRACT(WEEK FROM product.publication_date)", 8, "INT"), + ("EXTRACT('week_day' FROM product.publication_date)", 6, "INT"), + ("EXTRACT(WEEK_DAY FROM product.publication_date)", 6, "INT"), + ("EXTRACT('iso_week_day' FROM product.publication_date)", 5, "INT"), + ("EXTRACT(ISO_WEEK_DAY FROM product.publication_date)", 5, "INT"), + ( + "DATETRUNC('year', product.publication_date)", + datetime.date(2026, 1, 1), + "DATE", + ), + ( + "DATETRUNC('quarter', product.publication_date)", + datetime.date(2026, 1, 1), + "DATE", + ), + ( + "DATETRUNC('month', product.publication_date)", + datetime.date(2026, 2, 1), + "DATE", + ), + ( + "DATETRUNC('day', product.publication_date)", + datetime.date(2026, 2, 20), + "DATE", + ), + # Time functions + ("EXTRACT('hour' FROM product.category.closing_hour)", 22, "INT"), + ("EXTRACT(HOUR FROM product.category.closing_hour)", 22, "INT"), + ("EXTRACT('minute' FROM product.category.closing_hour)", 30, "INT"), + ("EXTRACT(MINUTE FROM product.category.closing_hour)", 30, "INT"), + ("EXTRACT('second' FROM product.category.closing_hour)", 0, "INT"), + ("EXTRACT(SECOND FROM product.category.closing_hour)", 0, "INT"), + ( + "DATETRUNC('hour', product.category.closing_hour)", + datetime.time(22, 0), + "TIME", + ), + ( + "DATETRUNC('minute', product.category.closing_hour)", + datetime.time(22, 30, 0), + "TIME", + ), + ( + "DATETRUNC('second', product.category.closing_hour)", + datetime.time(22, 30, 0), + "TIME", + ), # String functions ( "CONCAT(product.title, ' for ', order.customer.name)", @@ -338,14 +397,16 @@ def test_case_type_guess(engine_t1): ], ) def test_functions(engine_t1, expr, result, type_guess): - res = engine_t1.query( - f""" + res = list( + engine_t1.query( + f""" SELECT single_price, quantity, {expr} AS result, TYPE_INFO({expr}) AS type_guess FROM orderpositions WHERE quantity = 3 """ + ) ) - assert list(res) == [ + assert res == [ { "single_price": Decimal("19.00"), "quantity": 3, @@ -353,6 +414,9 @@ def test_functions(engine_t1, expr, result, type_guess): "type_guess": type_guess, }, ] + if type_guess == "DATE": + # date and datetime are quite compatible, let's be sure about what we get + assert type(res[0]["result"]) == datetime.date @pytest.mark.django_db @@ -484,3 +548,40 @@ def test_functions_with_timezone(engine_t1, expr, result): assert list(res) == [ {"single_price": Decimal("19.00"), "quantity": 3, "result": result}, ] + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "expr,result", + [ + ( + "EXTRACT('minute' FROM product.publication_date)", + "Unsupported extract value", + ), + ("EXTRACT('hour' FROM product.publication_date)", "Unsupported extract value"), + ( + "EXTRACT('second' FROM product.publication_date)", + "Unsupported extract value", + ), + ("DATETRUNC('hour', product.publication_date)", "Unsupported truncation type"), + ( + "DATETRUNC('minute', product.publication_date)", + "Unsupported truncation type", + ), + ( + "DATETRUNC('second', product.publication_date)", + "Unsupported truncation type", + ), + ], +) +def test_date_too_much_granularity(engine_t1, expr, result): + with pytest.raises(QueryNotSupported) as e: + engine_t1.query( + f""" + SELECT {expr} AS result + FROM orderpositions + WHERE quantity = 3 + """, + timezone=tz_ny, + ) + assert result in str(e) diff --git a/tests/testapp/factories.py b/tests/testapp/factories.py index a67b3a8..512b348 100644 --- a/tests/testapp/factories.py +++ b/tests/testapp/factories.py @@ -18,6 +18,7 @@ class Meta: class CategoryFactory(DjangoModelFactory): title = factory.Sequence(lambda n: "Category %d" % n) + closing_hour = datetime.time(hour=22, minute=30) class Meta: model = Category diff --git a/tests/testapp/models.py b/tests/testapp/models.py index dac6833..3149552 100644 --- a/tests/testapp/models.py +++ b/tests/testapp/models.py @@ -13,6 +13,7 @@ class Meta: class Category(models.Model): tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE) title = models.CharField(max_length=250) + closing_hour = models.TimeField() class Meta: ordering = ("id",) diff --git a/tests/testapp/ormql.py b/tests/testapp/ormql.py index f02f55c..be1616a 100644 --- a/tests/testapp/ormql.py +++ b/tests/testapp/ormql.py @@ -15,6 +15,7 @@ class Meta: columns = [ "id", "title", + "closing_hour", ]