feat: Add prepared/compiled/cached parametrized queries

tortoise/backends/mysql/executor.py

@@ -1,4 +1,6 @@
 import enum
+from collections.abc import Callable
+from typing import Any
 
 from pypika_tortoise import SqlContext, functions
 from pypika_tortoise.enums import SqlTypes
@@ -32,6 +34,7 @@
     search,
     starts_with,
 )
+from tortoise.parameter import Parameter
 
 
 class MySQLRegexpComparators(enum.Enum):
@@ -49,10 +52,47 @@ def get_value_sql(self, ctx: SqlContext) -> str:
         return format_quotes(value, quote_char)
 
 
+# TODO: maybe there is better way to do this?
+class StrParamWrapper(ValueWrapper):
+    value: Parameter
+
+    def get_value_sql(self, ctx: SqlContext) -> str:
+        quote_char = ctx.secondary_quote_char or ""
+        real_encoder = self.value.encode
+
+        def _encoder(val: str) -> str:
+            if real_encoder is not None:
+                val = real_encoder(val)
+            val = val.replace(quote_char, quote_char * 2)
+            return format_quotes(val, quote_char)
+
+        new_param = self.value.clone()
+        new_param.encode = _encoder
+        return new_param  # type: ignore[return-value]
+
+
 def escape_like(val: str) -> str:
     return val.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
 
 
+def _format_str_or_parameter(
+    value: str | Parameter,
+    like_start: bool = False,
+    like_end: bool = False,
+    escape_func: Callable[[Any], str] = escape_like,
+) -> Term:
+    like_at_start = "%" if like_start else ""
+    like_at_end = "%" if like_end else ""
+
+    if isinstance(value, Parameter):
+        value.encode = escape_func
+        if like_start or like_end:
+            value.encode = lambda val: f"{like_at_start}{escape_func(val)}{like_at_end}"
+        return StrParamWrapper(value)
+    else:
+        return StrWrapper(f"{like_at_start}{escape_func(value)}{like_at_end}")
+
+
 def mysql_contains(field: Term, value: str) -> Criterion:
     return Like(
         functions.Cast(field, SqlTypes.CHAR), StrWrapper(f"%{escape_like(value)}%"), escape=""
@@ -61,51 +101,59 @@ def mysql_contains(field: Term, value: str) -> Criterion:
 
 def mysql_starts_with(field: Term, value: str) -> Criterion:
     return Like(
-        functions.Cast(field, SqlTypes.CHAR), StrWrapper(f"{escape_like(value)}%"), escape=""
+        functions.Cast(field, SqlTypes.CHAR),
+        _format_str_or_parameter(value, False, True, escape_like),
+        escape="",
     )
 
 
 def mysql_ends_with(field: Term, value: str) -> Criterion:
     return Like(
-        functions.Cast(field, SqlTypes.CHAR), StrWrapper(f"%{escape_like(value)}"), escape=""
+        functions.Cast(field, SqlTypes.CHAR),
+        _format_str_or_parameter(value, True, False, escape_like),
+        escape="",
     )
 
 
 def mysql_insensitive_exact(field: Term, value: str) -> Criterion:
-    return functions.Upper(functions.Cast(field, SqlTypes.CHAR)).eq(functions.Upper(str(value)))
+    return functions.Upper(functions.Cast(field, SqlTypes.CHAR)).eq(
+        functions.Upper(_format_str_or_parameter(value, escape_func=str))
+    )
 
 
 def mysql_insensitive_contains(field: Term, value: str) -> Criterion:
     return Like(
         functions.Upper(functions.Cast(field, SqlTypes.CHAR)),
-        functions.Upper(StrWrapper(f"%{escape_like(value)}%")),
+        functions.Upper(_format_str_or_parameter(value, True, True, escape_like)),
         escape="",
     )
 
 
 def mysql_insensitive_starts_with(field: Term, value: str) -> Criterion:
     return Like(
         functions.Upper(functions.Cast(field, SqlTypes.CHAR)),
-        functions.Upper(StrWrapper(f"{escape_like(value)}%")),
+        functions.Upper(_format_str_or_parameter(value, False, True, escape_like)),
         escape="",
     )
 
 
 def mysql_insensitive_ends_with(field: Term, value: str) -> Criterion:
     return Like(
         functions.Upper(functions.Cast(field, SqlTypes.CHAR)),
-        functions.Upper(StrWrapper(f"%{escape_like(value)}")),
+        functions.Upper(_format_str_or_parameter(value, True, False, escape_like)),
         escape="",
     )
 
 
 def mysql_search(field: Term, value: str) -> SearchCriterion:
-    return SearchCriterion(field, expr=StrWrapper(value))
+    return SearchCriterion(field, expr=_format_str_or_parameter(value, escape_func=lambda x: x))
 
 
 def mysql_posix_regex(field: Term, value: str) -> BasicCriterion:
     return BasicCriterion(
-        MySQLRegexpComparators.REGEXP, Coalesce(Cast(field, SqlTypes.CHAR)), StrWrapper(value)
+        MySQLRegexpComparators.REGEXP,
+        Coalesce(Cast(field, SqlTypes.CHAR)),
+        _format_str_or_parameter(value, escape_func=lambda x: x),
     )
 
 

tortoise/expressions.py

@@ -22,8 +22,9 @@
 from tortoise.exceptions import FieldError, OperationalError
 from tortoise.fields.base import Field
 from tortoise.fields.data import JSONField
-from tortoise.fields.relational import RelationalField
+from tortoise.fields.relational import ForeignKeyFieldInstance, RelationalField
 from tortoise.filters import FilterInfoDict
+from tortoise.parameter import Parameter
 from tortoise.query_utils import (
     QueryModifier,
     TableCriterionTuple,
@@ -386,6 +387,9 @@ def _process_filter_kwarg(
         else:
             filter_info = model._meta.get_filter(key)
 
+        if isinstance(value, Parameter):
+            value.model = model
+
         field_object = None
         if "table" in filter_info:
             # join the table
@@ -395,15 +399,26 @@ def _process_filter_kwarg(
                 == filter_info["table"][filter_info["backward_key"]],
             )
             if "value_encoder" in filter_info:
-                value = filter_info["value_encoder"](value, model)
+                if isinstance(value, Parameter):
+                    value.value_encoder = filter_info["value_encoder"]
+                else:
+                    value = filter_info["value_encoder"](value, model)
             table = filter_info["table"]
         elif not isinstance(value, Term):
             field_object = model._meta.fields_map[filter_info["field"]]
-            value = (
-                filter_info["value_encoder"](value, model, field_object)
-                if "value_encoder" in filter_info
-                else field_object.to_db_value(value, model)
-            )
+            value_encoder = filter_info["value_encoder"] if "value_encoder" in filter_info else None
+            if isinstance(value, Parameter):
+                if isinstance(field_object.reference, ForeignKeyFieldInstance):
+                    fk_to_field = field_object.reference.to_field
+                    value.value_getter = lambda obj: getattr(obj, fk_to_field)
+                value.field_object = field_object
+                value.value_encoder = value_encoder
+            else:
+                value = (
+                    value_encoder(value, model, field_object)
+                    if value_encoder is not None
+                    else field_object.to_db_value(value, model)
+                )
         op = filter_info["operator"]
         term: Term = table[filter_info.get("source_field", filter_info["field"])]
         if field_object is not None:

tortoise/filters.py

@@ -21,6 +21,7 @@
 from tortoise.contrib.postgres.fields import ArrayField, TSVectorField
 from tortoise.fields import Field, JSONField
 from tortoise.fields.relational import BackwardFKRelation, ManyToManyFieldInstance
+from tortoise.parameter import CollectionParameter, Parameter
 
 if sys.version_info >= (3, 11):  # pragma：nocoverage
     from typing import NotRequired
@@ -102,6 +103,8 @@ def array_encoder(value: Any | Sequence[Any], instance: Model, field: Field) ->
 
 def is_in(field: Term, value: Any) -> Criterion:
     if value:
+        if isinstance(value, Parameter):
+            return CollectionParameter(field, value, True)
         return field.isin(value)
     # SQL has no False, so we return 1=0
     return BasicCriterion(
@@ -113,6 +116,8 @@ def is_in(field: Term, value: Any) -> Criterion:
 
 def not_in(field: Term, value: Any) -> Criterion:
     if value:
+        if isinstance(value, Parameter):
+            return CollectionParameter(field, value, False)
         return field.notin(value) | field.isnull()
     # SQL has no True, so we return 1=1
     return BasicCriterion(
@@ -142,8 +147,11 @@ def not_null(field: Term, value: Any) -> Criterion:
     return field.isnull()
 
 
-def contains(field: Term, value: str) -> Criterion:
-    return Like(Cast(field, SqlTypes.VARCHAR), field.wrap_constant(f"%{escape_like(value)}%"))
+def contains(field: Term, value: str | Parameter) -> Criterion:
+    return Like(
+        Cast(field, SqlTypes.VARCHAR),
+        field.wrap_constant(_format_str_or_parameter(field, value, True, True)),
+    )
 
 
 def search(field: Term, value: str) -> Any:
@@ -165,33 +173,58 @@ def insensitive_posix_regex(field: Term, value: str):
     )
 
 
-def starts_with(field: Term, value: str) -> Criterion:
-    return Like(Cast(field, SqlTypes.VARCHAR), field.wrap_constant(f"{escape_like(value)}%"))
+def _format_str_or_parameter(
+    field: Term,
+    value: str | Parameter,
+    like_start: bool = False,
+    like_end: bool = False,
+    escape_func: Callable[[Any], str] = escape_like,
+) -> Term:
+    like_at_start = "%" if like_start else ""
+    like_at_end = "%" if like_end else ""
 
+    if isinstance(value, Parameter):
+        value.encode = escape_func
+        wrapped = ValueWrapper(value)
+        if like_start or like_end:
+            value.encode = lambda val: f"{like_at_start}{escape_func(val)}{like_at_end}"
+        return wrapped
+    else:
+        return field.wrap_constant(f"{like_at_start}{escape_func(value)}{like_at_end}")
 
-def ends_with(field: Term, value: str) -> Criterion:
-    return Like(Cast(field, SqlTypes.VARCHAR), field.wrap_constant(f"%{escape_like(value)}"))
 
+def starts_with(field: Term, value: str | Parameter) -> Criterion:
+    return Like(Cast(field, SqlTypes.VARCHAR), _format_str_or_parameter(field, value, False, True))
 
-def insensitive_exact(field: Term, value: str) -> Criterion:
-    return Upper(Cast(field, SqlTypes.VARCHAR)).eq(Upper(str(value)))
 
+def ends_with(field: Term, value: str | Parameter) -> Criterion:
+    return Like(Cast(field, SqlTypes.VARCHAR), _format_str_or_parameter(field, value, True, False))
 
-def insensitive_contains(field: Term, value: str) -> Criterion:
+
+def insensitive_exact(field: Term, value: str | Parameter) -> Criterion:
+    return Upper(Cast(field, SqlTypes.VARCHAR)).eq(
+        Upper(_format_str_or_parameter(field, value, escape_func=str))
+    )
+
+
+def insensitive_contains(field: Term, value: str | Parameter) -> Criterion:
     return Like(
-        Upper(Cast(field, SqlTypes.VARCHAR)), field.wrap_constant(Upper(f"%{escape_like(value)}%"))
+        Upper(Cast(field, SqlTypes.VARCHAR)),
+        field.wrap_constant(Upper(_format_str_or_parameter(field, value, True, True))),
     )
 
 
-def insensitive_starts_with(field: Term, value: str) -> Criterion:
+def insensitive_starts_with(field: Term, value: str | Parameter) -> Criterion:
     return Like(
-        Upper(Cast(field, SqlTypes.VARCHAR)), field.wrap_constant(Upper(f"{escape_like(value)}%"))
+        Upper(Cast(field, SqlTypes.VARCHAR)),
+        field.wrap_constant(Upper(_format_str_or_parameter(field, value, False, True))),
     )
 
 
-def insensitive_ends_with(field: Term, value: str) -> Criterion:
+def insensitive_ends_with(field: Term, value: str | Parameter) -> Criterion:
     return Like(
-        Upper(Cast(field, SqlTypes.VARCHAR)), field.wrap_constant(Upper(f"%{escape_like(value)}"))
+        Upper(Cast(field, SqlTypes.VARCHAR)),
+        field.wrap_constant(Upper(_format_str_or_parameter(field, value, True, False))),
     )
 
 
@@ -240,7 +273,7 @@ def json_contained_by(field: Term, value: str) -> Criterion:
 
 
 def json_filter(field: Term, value: dict) -> Criterion:
-    raise NotImplementedError("must be overridden in each xecutor")
+    raise NotImplementedError("must be overridden in each executor")
 
 
 def array_contains(field: Term, value: Any | Sequence[Any]) -> Criterion:

tortoise/models.py

@@ -50,6 +50,7 @@
     QuerySetSingle,
     RawSQLQuery,
 )
+from tortoise.queryset_compiled import BaseCompiledQuery
 from tortoise.router import router
 from tortoise.signals import Signals
 from tortoise.transactions import in_transaction
@@ -215,6 +216,7 @@ class MetaInfo:
         "db_complex_fields",
         "_default_ordering",
         "_ordering_validated",
+        "query_cache",
     )
 
     def __init__(self, meta: Model.Meta) -> None:
@@ -255,6 +257,7 @@ def __init__(self, meta: Model.Meta) -> None:
         self.db_native_fields: list[tuple[str, str, Field]] = []
         self.db_default_fields: list[tuple[str, str, Field]] = []
         self.db_complex_fields: list[tuple[str, str, Field]] = []
+        self.query_cache: dict[str, BaseCompiledQuery] = {}
 
     @property
     def full_name(self) -> str:
@@ -1610,6 +1613,10 @@ async def fetch_for_list(
         db = using_db or cls._choose_db()
         await db.executor_class(model=cls, db=db).fetch_for_list(instance_list, *args)
 
+    @classmethod
+    def remove_compiled_query(cls, key: str) -> None:
+        cls._meta.query_cache.pop(key, None)
+
     @classmethod
     def _check(cls) -> None:
         """