Add schema-aware dynamic search and native Expr composition

examples/school-management/app/main.py

@@ -12,6 +12,41 @@
 from runtime_module import GENERATED_RUNTIME_MODULE
 from teaql.data_service import SQLiteTeaQLClient
 from teaql.runtime import UserContext
+from teaql.core.dynamic_search import normalize_dynamic_search
+
+
+async def verify_dynamic_search(context):
+    models = {
+        "School": {"fields": {"name": "string"}, "relations": {"platform": "Platform"}},
+        "Platform": {"fields": {"name": "string"}, "relations": {}},
+    }
+    populated = {"filter": {"name": "Riverside Primary School", "platform.name": "Deployment Campus",
+        "removed": "SECRET_VALUE", "platform.removed": "SECRET_VALUE"},
+        "orderBy": [{"field": "removed", "direction": "asc"}]}
+    for authorized_platform in (1, 2):
+        for search_input in (populated, {}):
+            # Authorization remains present even when the entire search form is absent.
+            request = Q.schools().with_name_is("Riverside Primary School").with_platform_matching(
+                Q.platforms().with_id_is(authorized_platform))
+            search, warnings = normalize_dynamic_search(search_input, "School", models, lambda _: None)
+            for field, predicate in search["filter"].items():
+                value = predicate.get("$eq")
+                if not isinstance(value, str):
+                    raise ValueError("Demo binding supports string equality only")
+                if field == "name":
+                    request = request.with_name_is(value)
+                elif field == "platform.name":
+                    request = request.with_platform_matching(Q.platforms().with_name_is(value))
+                else:
+                    raise ValueError("Missing trusted demo binding")
+            rows = await (request.order_by_id_descending().limit(2)
+                .comment("what: generated School dynamic search")
+                .purpose("why: retain related authorization with stale or absent search fields")
+                .execute_for_list(context))
+            assert len(rows) == (1 if authorized_platform == 1 else 0)
+            assert len(warnings) == (3 if search_input is populated else 0)
+            assert "SECRET_VALUE" not in str(warnings)
+    print("PASS Python generated School dynamic search: independent related scope and typed Q bindings")
 
 
 async def main() -> None:
@@ -70,6 +105,8 @@ async def main() -> None:
     school.update_active(True)
     await school.audit_as("Create Riverside Primary School").save(context)
 
+    await verify_dynamic_search(context)
+
     loaded = await (Q.schools().with_id_is(school.id)
         .select_platform_with(Q.platforms_minimal().select_name().select_base_url())
         .select_school_type_with(Q.school_types_minimal().select_name().select_code().select_display_order())

examples/school-management/requests/platform_request.py

@@ -10,6 +10,7 @@
 )
 from models.platform import Platform
 from typing import Protocol
+from copy import deepcopy
 
 class QuerySelection(Protocol):
     query: SelectQuery
@@ -545,13 +546,15 @@ def have_no_school_types(self):
         return self.without_school_type_list_matching(SchoolTypeRequest())
 
     def with_school_type_list_matching(self, child_request):
-        self.query.and_filter(in_subquery(column("id"), "SchoolType", child_request.query))
-        child_request.query._projection = ["platform"]
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["platform"]
+        self.query.and_filter(in_subquery(column("id"), "SchoolType", child_query))
         return self
 
     def without_school_type_list_matching(self, child_request):
-        self.query.and_filter(not_in_subquery(column("id"), "SchoolType", child_request.query))
-        child_request.query._projection = ["platform"]
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["platform"]
+        self.query.and_filter(not_in_subquery(column("id"), "SchoolType", child_query))
         return self
     def have_schools(self):
         from requests.school_request import SchoolRequest
@@ -562,13 +565,15 @@ def have_no_schools(self):
         return self.without_school_list_matching(SchoolRequest())
 
     def with_school_list_matching(self, child_request):
-        self.query.and_filter(in_subquery(column("id"), "School", child_request.query))
-        child_request.query._projection = ["platform"]
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["platform"]
+        self.query.and_filter(in_subquery(column("id"), "School", child_query))
         return self
 
     def without_school_list_matching(self, child_request):
-        self.query.and_filter(not_in_subquery(column("id"), "School", child_request.query))
-        child_request.query._projection = ["platform"]
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["platform"]
+        self.query.and_filter(not_in_subquery(column("id"), "School", child_query))
         return self
     def count_school_types(self):
         return self.count_school_types_as("count_school_types")

examples/school-management/requests/school_request.py

@@ -10,6 +10,7 @@
 )
 from models.school import School
 from typing import Protocol
+from copy import deepcopy
 
 class QuerySelection(Protocol):
     query: SelectQuery
@@ -127,13 +128,15 @@ def select_school_type_with(self, child_request):
         self.query.relation_query("school_type", child_request.query)
         return self
     def with_platform_matching(self, child_request):
-        child_request.query._projection = ["id"]
-        self.query.and_filter(in_subquery(column("platform"), "Platform", child_request.query))
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["id"]
+        self.query.and_filter(in_subquery(column("platform"), "Platform", child_query))
         return self
 
     def without_platform_matching(self, child_request):
-        child_request.query._projection = ["id"]
-        self.query.and_filter(not_in_subquery(column("platform"), "Platform", child_request.query))
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["id"]
+        self.query.and_filter(not_in_subquery(column("platform"), "Platform", child_query))
         return self
 
     def have_platform(self):
@@ -144,13 +147,15 @@ def have_no_platform(self):
         self.query.and_filter(is_null(column("platform")))
         return self
     def with_school_type_matching(self, child_request):
-        child_request.query._projection = ["id"]
-        self.query.and_filter(in_subquery(column("school_type"), "SchoolType", child_request.query))
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["id"]
+        self.query.and_filter(in_subquery(column("school_type"), "SchoolType", child_query))
         return self
 
     def without_school_type_matching(self, child_request):
-        child_request.query._projection = ["id"]
-        self.query.and_filter(not_in_subquery(column("school_type"), "SchoolType", child_request.query))
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["id"]
+        self.query.and_filter(not_in_subquery(column("school_type"), "SchoolType", child_query))
         return self
 
     def have_school_type(self):

examples/school-management/requests/school_type_request.py

@@ -10,6 +10,7 @@
 )
 from models.school_type import SchoolType
 from typing import Protocol
+from copy import deepcopy
 
 class QuerySelection(Protocol):
     query: SelectQuery
@@ -105,13 +106,15 @@ def select_platform_with(self, child_request):
         self.query.relation_query("platform", child_request.query)
         return self
     def with_platform_matching(self, child_request):
-        child_request.query._projection = ["id"]
-        self.query.and_filter(in_subquery(column("platform"), "Platform", child_request.query))
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["id"]
+        self.query.and_filter(in_subquery(column("platform"), "Platform", child_query))
         return self
 
     def without_platform_matching(self, child_request):
-        child_request.query._projection = ["id"]
-        self.query.and_filter(not_in_subquery(column("platform"), "Platform", child_request.query))
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["id"]
+        self.query.and_filter(not_in_subquery(column("platform"), "Platform", child_query))
         return self
 
     def have_platform(self):
@@ -555,13 +558,15 @@ def have_no_schools(self):
         return self.without_school_list_matching(SchoolRequest())
 
     def with_school_list_matching(self, child_request):
-        self.query.and_filter(in_subquery(column("id"), "School", child_request.query))
-        child_request.query._projection = ["school_type"]
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["school_type"]
+        self.query.and_filter(in_subquery(column("id"), "School", child_query))
         return self
 
     def without_school_list_matching(self, child_request):
-        self.query.and_filter(not_in_subquery(column("id"), "School", child_request.query))
-        child_request.query._projection = ["school_type"]
+        child_query = deepcopy(child_request.query)
+        child_query.projection = ["school_type"]
+        self.query.and_filter(not_in_subquery(column("id"), "School", child_query))
         return self
     def count_schools(self):
         return self.count_schools_as("count_schools")

src/teaql/core/dynamic_search.py

@@ -0,0 +1,135 @@
+"""Local UI-search input; not a permissive TFP decoder or authorization policy."""
+import json
+import logging
+import math
+import re
+from copy import deepcopy
+from datetime import date
+from .expr import Expr
+from .query import OrderBy
+
+_LOG = logging.getLogger(__name__)
+_OPS = {'$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$notIn', '$contains'}
+
+
+def _reject_json_constant(_):
+    raise ValueError('Non-standard JSON number')
+
+
+def _warning(warning):
+    _LOG.warning('%s entity=%s clause=%s fieldPath=%s', warning['code'],
+                 warning['entity'], warning['clause'], warning['fieldPath'])
+
+
+def normalize_dynamic_search(source, entity, models, warn=_warning, max_clauses=100):
+    """models and limits come from trusted setup, never the submitted JSON."""
+    if type(max_clauses) is not int or max_clauses < 1:
+        raise ValueError('Invalid search limit')
+    if entity not in models:
+        raise ValueError('Unknown search entity')
+    try:
+        value = json.loads(source, parse_constant=_reject_json_constant) if isinstance(source, str) else source
+    except (ValueError, TypeError):
+        raise ValueError('Dynamic search requires valid JSON') from None
+    if not isinstance(value, dict) or set(value) - {'filter', 'orderBy'}:
+        raise ValueError('Unsupported dynamic search input or control')
+    filters, orders = value.get('filter', {}), value.get('orderBy', [])
+    if not isinstance(filters, dict) or not isinstance(orders, list):
+        raise ValueError('Invalid search filter or ordering')
+    if len(filters) + len(orders) > max_clauses:
+        raise ValueError('Dynamic search exceeds clause limit')
+    result, warnings = {'filter': {}, 'orderBy': []}, []
+
+    def field_type(path):
+        if not isinstance(path, str):
+            raise ValueError('Invalid search field path')
+        parts = path.split('.')
+        if len(parts) > 16 or any(not p or p.startswith('$') or p in
+                                 {'__proto__', 'prototype', 'constructor'} for p in parts):
+            raise ValueError('Invalid search field path')
+        model = models[entity]
+        for part in parts[:-1]:
+            target = model['relations'].get(part)
+            if target is None:
+                return None
+            if target not in models:
+                raise ValueError('Invalid trusted search relation metadata')
+            model = models[target]
+        return model['fields'].get(parts[-1])
+
+    def missing(path, clause):
+        warnings.append(dict(code='DYNAMIC_SEARCH_UNKNOWN_FIELD', entity=entity,
+                             clause=clause, fieldPath=path))
+
+    def scalar(item, kind):
+        if item is None:
+            return
+        valid = False
+        if kind in ('integer', 'timestamp'):
+            valid = (type(item) in (int, float) and abs(item) <= 9007199254740991
+                     and math.isfinite(item) and item == int(item))
+        elif kind == 'number':
+            valid = type(item) in (int, float) and math.isfinite(item)
+        elif kind == 'string':
+            valid = isinstance(item, str)
+        elif kind == 'boolean':
+            valid = type(item) is bool
+        elif kind == 'decimal':
+            valid = (isinstance(item, str) and re.fullmatch(r'[+-]?[0-9]+(?:\.[0-9]+)?', item) is not None
+                     or type(item) in (int, float) and math.isfinite(item))
+        elif kind == 'date' and isinstance(item, str) and re.fullmatch(r'\d{4}-\d{2}-\d{2}', item):
+            try:
+                valid = date.fromisoformat(item).isoformat() == item
+            except ValueError:
+                pass
+        if not valid:
+            raise ValueError('Invalid value for known search field')
+
+    for path, predicate in filters.items():
+        predicate = predicate if isinstance(predicate, dict) else {'$eq': predicate}
+        if len(predicate) != 1 or next(iter(predicate)) not in _OPS:
+            raise ValueError('Unsupported or malformed dynamic search operator')
+        operator, item = next(iter(predicate.items()))
+        if operator in ('$in', '$notIn') and (not isinstance(item, list) or len(item) > 1000):
+            raise ValueError('Invalid or oversized search value list')
+        kind = field_type(path)
+        if kind is None:
+            missing(path, 'FILTER')
+            continue
+        if operator == '$contains' and kind != 'string':
+            raise ValueError('String operator requires a string field')
+        if isinstance(item, list):
+            if operator not in ('$in', '$notIn'):
+                raise ValueError('Unexpected search value list')
+            for element in item:
+                scalar(element, kind)
+        else:
+            scalar(item, kind)
+        result['filter'][path] = deepcopy(predicate)
+    for order in orders:
+        if (not isinstance(order, dict) or set(order) != {'field', 'direction'}
+                or order['direction'] not in ('asc', 'desc')):
+            raise ValueError('Invalid dynamic search ordering')
+        if field_type(order['field']) is None:
+            missing(order['field'], 'ORDER_BY')
+        else:
+            result['orderBy'].append(dict(order))
+    for warning in warnings:
+        warn(dict(warning))
+    return result, warnings
+
+
+def merge_dynamic_search(base, source, models, filter_binding, order_binding, warn=_warning):
+    """Bindings are trusted native-API adapters; preserve the original scoped request."""
+    search, warnings = normalize_dynamic_search(source, base.entity, models, lambda _: None)
+    filters = [filter_binding(path, predicate) for path, predicate in search['filter'].items()]
+    orders = [order_binding(order['field'], order['direction']) for order in search['orderBy']]
+    if any(not isinstance(expr, Expr) for expr in filters) or any(not isinstance(order, OrderBy) for order in orders):
+        raise ValueError('Invalid trusted search binding')
+    query = deepcopy(base)
+    for expr in filters:
+        query.filter_expr = Expr.new_and(query.filter_expr, expr) if query.filter_expr is not None else expr
+    query.order_by_items.extend(orders)
+    for warning in warnings:
+        warn(dict(warning))
+    return query, warnings

src/teaql/core/query.py

@@ -2,7 +2,8 @@
 from typing import List, Optional, Any, Dict
 from copy import deepcopy
 from dataclasses import dataclass, field
-from .expr import Expr, ExprBuilder
+from .expr import (AndExpr, BinaryExpr, BinaryOp, ColumnExpr, Expr, ExprBuilder,
+                   ValueExpr)
 from .mutation import TraceNode
 
 class SortDirection(Enum):
@@ -101,6 +102,32 @@ class ObjectGroupBy:
     storage_field: str
     query: 'SelectQuery'
 
+
+def _is_default_live_filter(expr: Optional[Expr]) -> bool:
+    return (isinstance(expr, BinaryExpr)
+            and isinstance(expr.left, ColumnExpr)
+            and expr.left.name == "version"
+            and expr.op is BinaryOp.Gte
+            and isinstance(expr.right, ValueExpr)
+            and expr.right.value.val == 1)
+
+
+def _remove_default_live_filter(expr: Optional[Expr]) -> Optional[Expr]:
+    if expr is None or _is_default_live_filter(expr):
+        return None
+    if isinstance(expr, AndExpr):
+        retained = []
+        for child in expr.exprs:
+            child = _remove_default_live_filter(child)
+            if child is not None:
+                retained.append(child)
+        if not retained:
+            return None
+        if len(retained) == 1:
+            return retained[0]
+        return AndExpr(retained)
+    return expr
+
 @dataclass
 class FacetRequest:
     name: str
@@ -257,6 +284,21 @@ def and_filter(self, expr: Expr) -> 'SelectQuery':
         else:
             self.filter_expr = expr
         return self
+
+    def with_deleted_rows(self) -> 'SelectQuery':
+        """Remove only TeaQL's implicit live-row predicate.
+
+        Generated requests add ``version >= 1`` during construction.  Deletion
+        helpers must not reach into private fields or discard application-owned
+        filters when opting into tombstones.
+        """
+        self.filter_expr = _remove_default_live_filter(self.filter_expr)
+        return self
+
+    def deleted_rows_only(self) -> 'SelectQuery':
+        self.with_deleted_rows()
+        self.and_filter(Expr.lte("version", -1))
+        return self
 
     def order_asc(self, field: str) -> 'SelectQuery':
         self.order_by_items.append(OrderBy.asc(field))