Align SmartList semantics and typed query boundaries

src/teaql/core/list.py

@@ -1,7 +1,10 @@
 from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, TypeVar
 from dataclasses import dataclass
+from copy import deepcopy
 
 T = TypeVar("T")
+U = TypeVar("U")
+K = TypeVar("K")
 
 @dataclass(frozen=True)
 class TeaQLPage(Generic[T]):
@@ -11,31 +14,83 @@ class TeaQLPage(Generic[T]):
     limit: int
 
 class SmartList(List[T], Generic[T]):
-    def __init__(self, data: Iterable[T] = (), facets: Optional[Dict[str, Any]] = None,
-                 total_count: Optional[int] = None, aggregations: Optional[Dict[str, Any]] = None):
+    """Typed rows plus a source-query metadata snapshot.
+
+    Construction copies metadata recursively; entity rows retain normal list semantics.
+    In-place edits retain the source snapshot. Derived filter/flat_map clear it.
+    """
+    def __init__(self, data: Iterable[T] = (),
+                 facets: Optional[Dict[str, 'SmartList[Dict[str, Any]]']] = None,
+                 total_count: Optional[int] = None, aggregations: Optional[Dict[str, Any]] = None,
+                 *, summary: Optional[Dict[str, Any]] = None):
         super().__init__(data)
-        self.facets = facets or {}
-        self.total_count = total_count if total_count is not None else len(self)
-        self.aggregations = dict(aggregations or {})
+        if total_count is not None and (isinstance(total_count, bool)
+                or not isinstance(total_count, int) or total_count < 0):
+            raise ValueError("total_count must be an exact non-negative integer")
+        self.facets: Dict[str, SmartList[Dict[str, Any]]] = {
+            name: deepcopy(facet) if isinstance(facet, SmartList) else SmartList(facet)
+            for name, facet in (facets or {}).items()
+        }
+        self.total_count = total_count
+        self.aggregations = deepcopy(aggregations or {})
+        self.summary = deepcopy(summary or {})
+        self._is_loaded = True
+
+    @classmethod
+    def empty(cls) -> 'SmartList[T]':
+        result = cls()
+        result._is_loaded = False
+        return result
+
+    @property
+    def is_loaded(self) -> bool:
+        return self._is_loaded
+
+    def total_count_or_len(self) -> int:
+        return self.total_count if self.total_count is not None else len(self)
 
     @property
     def data(self) -> 'SmartList[T]':
         return self
 
-    def facet(self, name: str) -> Any:
+    def facet(self, name: str) -> Optional['SmartList[Dict[str, Any]]']:
         return self.facets.get(name)
 
-    def map(self, f: Callable[[T], Any]) -> 'SmartList[Any]':
-        return SmartList((f(x) for x in self), self.facets, self.total_count, self.aggregations)
-
-    def filter(self, f) -> 'SmartList':
-        return SmartList((x for x in self if f(x)), self.facets, self.total_count, self.aggregations)
-
-    def flat_map(self, f) -> 'SmartList':
-        result = []
-        for x in self:
-            result.extend(f(x))
-        return SmartList(result, self.facets, self.total_count, self.aggregations)
+    def remove_facet(self, name: str) -> Optional['SmartList[Dict[str, Any]]']:
+        return self.facets.pop(name, None)
+
+    def take_facets(self) -> Dict[str, 'SmartList[Dict[str, Any]]']:
+        facets, self.facets = self.facets, {}
+        return facets
+
+    def map(self, f: Callable[[T], U]) -> 'SmartList[U]':
+        result = SmartList((f(x) for x in self), self.facets, self.total_count,
+                           self.aggregations, summary=self.summary)
+        result._is_loaded = self._is_loaded
+        return result
+
+    def filter(self, f: Callable[[T], bool]) -> 'SmartList[T]':
+        result = SmartList(x for x in self if f(x))
+        result._is_loaded = self._is_loaded
+        return result
+
+    def flat_map(self, f: Callable[[T], Iterable[U]]) -> 'SmartList[U]':
+        result = SmartList(value for x in self for value in f(x))
+        result._is_loaded = self._is_loaded
+        return result
+
+    def merge_by(self, incoming: Iterable[T], key: Callable[[T], K]) -> None:
+        positions = {key(item): index for index, item in enumerate(self)}
+        for item in incoming:
+            item_key = key(item)
+            if item_key in positions:
+                self[positions[item_key]] = item
+            else:
+                positions[item_key] = len(self)
+                self.append(item)
+
+    def identity_map(self, key: Callable[[T], K]) -> Dict[K, T]:
+        return {key(item): item for item in self}
 
     def first(self) -> Any:
         return self[0] if self else None
@@ -63,8 +118,8 @@ def to_list(lst: SmartList) -> List[Any]:
 def to_set(lst: SmartList) -> set:
     return set(lst.data)
 
-def identity_map(lst: SmartList) -> Dict[Any, Any]:
-    return {x: x for x in lst.data}
+def identity_map(lst: SmartList[T], key_func: Callable[[T], K] = lambda value: value) -> Dict[K, T]:
+    return lst.identity_map(key_func)
 
 def group_by(lst: SmartList, key_func) -> Dict[Any, List[Any]]:
     result = {}
@@ -84,7 +139,7 @@ def ids(lst: SmartList) -> List[Any]:
     return [x.get('id') if isinstance(x, dict) else getattr(x, 'id', None) for x in lst.data]
 
 def map_by_id(lst: SmartList) -> Dict[Any, Any]:
-    return { (x.get('id') if isinstance(x, dict) else getattr(x, 'id', None)): x for x in lst.data }
+    return {identifier: item for identifier, item in zip(ids(lst), lst) if identifier is not None}
 
 def versions(lst: SmartList) -> List[Any]:
     return [x.get('version') if isinstance(x, dict) else getattr(x, 'version', None) for x in lst.data]

src/teaql/runtime/query_list.py

@@ -50,14 +50,18 @@ async def execute_typed_list(context, request: QueryRequest, hydrate: Callable[[
     service = context.require_resource('dataService')
 
     async def execute():
+        if root.query.group_by_items:
+            raise ValueError('grouped projections require a separate projection result boundary')
         total = None
+        row_query = root.query
         if count_operation is None:
             row_result = await service.query(context, root)
         else:
             alias = count_alias if count_alias is not None else '__teaql_total'
             query = deepcopy(root.query)
             query.aggregates = [item for item in query.aggregates
                                 if not (item.function == AggregateFunction.Count and item.alias == count_alias)]
+            row_query = query
             rows_request = root.with_query(query)
 
             async def count():
@@ -75,8 +79,17 @@ async def count():
             else:
                 total = await count()
                 row_result = await service.query(context, rows_request)
-        return SmartList((hydrate(row) for row in row_result.rows), facets=row_result.facets,
-                         total_count=total, aggregations={count_alias: total} if count_alias is not None else {})
+        aggregations = {count_alias: total} if count_alias is not None else {}
+        if row_query.aggregates:
+            if len(row_result.rows) != 1:
+                raise RuntimeError('dataService did not return one scalar aggregation result')
+            values = row_result.rows[0]
+            aggregations.update({item.alias: values[item.alias] for item in row_query.aggregates
+                                 if item.alias in values})
+            rows = ()
+        else:
+            rows = (hydrate(row) for row in row_result.rows)
+        return SmartList(rows, facets=row_result.facets, total_count=total, aggregations=aggregations)
 
     return await observe_query_operation(context, service, root, execute,
                                          lambda result: {'teaql.result.cardinality': len(result)})

src/teaql/sql/executor.py

@@ -1107,14 +1107,20 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac
                                           'success', affected_rows=affected_rows, intent_privacy=privacy, correlation=statement.correlation)
 
         generated_values = {}
-        if op == "insert" and last_insert_id:
-            # Assumes the first ID column
+        if op == "insert":
             id_prop = next((
                 p for p in getattr(entity_desc, 'properties', [])
                 if getattr(p, '_is_id', False) or getattr(p, 'is_id_val', False)
             ), None)
             if id_prop:
-                generated_values[id_prop.name] = Value.val_u64(last_insert_id)
+                # TeaQL assigned the modeled identity before compiling INSERT.
+                # SQLite lastrowid is a different key for BIGINT PRIMARY KEY;
+                # it is only an identity fallback when no assigned key exists.
+                assigned = req_data.values.get(id_prop.name)
+                if assigned is not None:
+                    generated_values[id_prop.name] = assigned
+                elif last_insert_id:
+                    generated_values[id_prop.name] = Value.val_u64(last_insert_id)
 
         id_prop = next(
             (

src/teaql/web_integration/fastapi/__init__.py

@@ -11,7 +11,7 @@ class WebResponse(BaseModel, Generic[T]):
     resultCode: int = 0
     status: str = "YES"
     message: Optional[str] = None
-    recordCount: int = 0
+    recordCount: Optional[int] = 0
     facets: Optional[Dict[str, Any]] = None
     version: str = "1.001"
     traceId: Optional[str] = None

tests/core/test_smart_list_semantics.py

@@ -0,0 +1,130 @@
+"""Behavioral vectors mirrored from conformance; no provider mocks certify SQLite."""
+from decimal import Decimal
+import json
+from pathlib import Path
+
+import pytest
+
+from teaql.core.list import SmartList, group_by, identity_map, ids, map_by_id, versions
+
+VECTORS = json.loads((Path(__file__).parents[1] / 'fixtures/smart-list-semantics.json').read_text())
+
+
+def source_list():
+    values = VECTORS['metadata']
+    return SmartList(VECTORS['transform']['rows'], total_count=VECTORS['transform']['total'],
+                     aggregations=values['aggregations'], summary=values['summary'],
+                     facets={name: SmartList(rows) for name, rows in values['facets'].items()})
+
+
+@pytest.mark.parametrize('case', VECTORS['states'], ids=lambda case: case['name'])
+def test_state_and_exact_count(case):
+    result = (SmartList(case['rows'], total_count=case['total'])
+              if case['loaded'] else SmartList.empty())
+    assert list(result) == case['rows']
+    assert result.is_loaded is case['loaded']
+    assert result.total_count == case['total']
+    assert result.total_count_or_len() == (len(result) if case['total'] is None else case['total'])
+    assert result.total_count == case['total'], 'fallback must not manufacture exact count'
+
+
+@pytest.mark.parametrize('value', [-1, 1.5, float('nan'), float('inf'), True, '3'])
+def test_invalid_count_is_rejected(value):
+    with pytest.raises(ValueError, match='exact non-negative integer'):
+        SmartList(total_count=value)
+
+
+def test_map_preserves_query_metadata_and_recursively_isolates_mutations():
+    source = source_list()
+    decimal = Decimal(VECTORS['metadata']['decimal']['value'])
+    source.aggregations['decimal'] = decimal
+    mapped = source.map(lambda item: item * 10)
+    assert mapped == VECTORS['transform']['mapTimesTen']
+    assert mapped.total_count == source.total_count and mapped.is_loaded
+    assert mapped.aggregations == source.aggregations
+    assert mapped.summary == source.summary and mapped.facets == source.facets
+    assert mapped.aggregations['decimal'] == decimal
+    assert mapped.aggregations['nullable'] is None and 'missing' not in mapped.aggregations
+
+    mapped.aggregations['nested']['values'].append(99)
+    mapped.summary['nested']['values'].append(99)
+    mapped.facets['status'][0]['count'] = 99
+    mapped.facets['status'].summary['new'] = True
+    mapped.facets.pop('empty')
+    assert source.aggregations['nested']['values'] == [1, 2]
+    assert source.summary['nested']['values'] == [3]
+    assert source.facets['status'][0]['count'] == 3
+    assert source.facets['status'].summary == {} and 'empty' in source.facets
+
+
+def test_constructor_owns_metadata_and_native_conversion_owns_positions():
+    metadata = {'nested': [1]}
+    facet = SmartList([{'values': [2]}])
+    source = SmartList([1, 2], aggregations=metadata, facets={'nested': facet})
+    metadata['nested'].append(3)
+    facet[0]['values'].append(4)
+    native = source.into_vec()
+    native[0] = 9
+    assert source == [1, 2]
+    assert source.aggregations == {'nested': [1]}
+    assert source.facets['nested'][0] == {'values': [2]}
+
+
+def test_cardinality_changing_derivations_clear_source_statistics_even_if_rows_match():
+    source = source_list()
+    filtered = source.filter(lambda item: item % 2 == 1)
+    expanded = source.flat_map(lambda item: [item, item])
+    assert filtered == VECTORS['transform']['filterOdd']
+    assert expanded == VECTORS['transform']['flatMapTwice']
+    for result in (filtered, expanded, source.filter(lambda item: True)):
+        assert result.total_count is None
+        assert result.aggregations == result.summary == result.facets == {}
+        assert result.is_loaded
+    assert source.total_count == 9 and len(source) == 3
+    for result in (SmartList.empty().map(str), SmartList.empty().filter(bool),
+                   SmartList.empty().flat_map(lambda item: [item])):
+        assert not result.is_loaded
+
+
+def test_merge_duplicate_positions_index_group_and_in_place_snapshot():
+    case = VECTORS['merge']
+    source = SmartList(case['target'], total_count=9, summary={'original': True})
+    source.merge_by(case['incoming'], lambda item: item['key'])
+    assert source == case['expected']
+    assert source.total_count == 9 and source.summary == {'original': True}
+    index = identity_map(source, lambda item: item['key'])
+    assert index['a'] == {'key': 'a', 'value': 4}
+    assert group_by(source, lambda item: item['key'])['a'] == [source[0], source[1]]
+    source.retain(lambda item: item['key'] == 'a')
+    assert len(source) == 2 and source.total_count == 9
+    assert identity_map(SmartList([1, 2, 1])) == {1: 1, 2: 2}
+
+
+def test_ids_versions_and_missing_id_indexing():
+    case = VECTORS['identity']
+    source = SmartList(case['rows'])
+    assert ids(source) == case['ids'] and versions(source) == case['versions']
+    index = map_by_id(source)
+    assert None not in index and len(index) == 4
+    assert [index[key] for key in (0, 1, '1', '')] == [source[i] for i in case['indexedPositions']]
+
+
+def test_safe_access_and_facets_transfer():
+    source = source_list()
+    assert all(source.get(index) is None for index in VECTORS['transform']['safeMissingIndexes'])
+    assert source.first() == 1 and source.last() == 3
+    assert SmartList().first() is None and SmartList().last() is None
+    assert source.facet('missing') is None
+    assert source.facet('empty') == [] and source.facet('empty').is_loaded
+    assert source.remove_facet('missing') is None
+    removed = source.remove_facet('empty')
+    assert removed == [] and source.facet('empty') is None
+    taken = source.take_facets()
+    assert 'status' in taken and source.facets == {}
+
+
+def test_loaded_state_has_no_new_public_write_entry():
+    with pytest.raises(AttributeError):
+        SmartList().is_loaded = False
+    with pytest.raises(TypeError):
+        SmartList(is_loaded=False)

tests/fixtures/smart-list-semantics.json

@@ -0,0 +1,34 @@
+{
+  "schema": "teaql.smart-list-semantics/v1",
+  "states": [
+    {"name": "unloaded", "loaded": false, "rows": [], "total": null},
+    {"name": "loaded-empty-unknown", "loaded": true, "rows": [], "total": null},
+    {"name": "loaded-empty-zero", "loaded": true, "rows": [], "total": 0},
+    {"name": "page", "loaded": true, "rows": [1, 2], "total": 5}
+  ],
+  "metadata": {
+    "aggregations": {"count": 5, "nullable": null, "nested": {"values": [1, 2]}},
+    "summary": {"label": "orders", "active": true, "nested": {"values": [3]}},
+    "facets": {"empty": [], "status": [{"status": "paid", "count": 3}]},
+    "decimal": {"type": "decimal", "value": "9007199254740993.125"}
+  },
+  "transform": {
+    "rows": [1, 2, 3], "total": 9,
+    "mapTimesTen": [10, 20, 30], "filterOdd": [1, 3], "flatMapTwice": [1, 1, 2, 2, 3, 3],
+    "safeMissingIndexes": [-1, 3, 100]
+  },
+  "merge": {
+    "target": [{"key": "a", "value": 1}, {"key": "a", "value": 2}, {"key": "b", "value": 3}],
+    "incoming": [{"key": "a", "value": 4}, {"key": "c", "value": 5}, {"key": "c", "value": 6}, {"key": "d", "value": 7}],
+    "expected": [{"key": "a", "value": 1}, {"key": "a", "value": 4}, {"key": "b", "value": 3}, {"key": "c", "value": 6}, {"key": "d", "value": 7}]
+  },
+  "identity": {
+    "rows": [{"id": 0, "version": 1}, {"id": null, "version": 2}, {"id": 1, "version": 3}, {"id": 1, "version": 4}, {"id": "1", "version": 5}, {"id": "", "version": 6}],
+    "ids": [0, null, 1, 1, "1", ""], "versions": [1, 2, 3, 4, 5, 6], "indexedPositions": [0, 3, 4, 5]
+  },
+  "sqlite": {
+    "rows": [{"id": 1, "status": "paid", "amount": "10.25"}, {"id": 2, "status": "open", "amount": "2.50"}, {"id": 3, "status": "paid", "amount": "7.25"}],
+    "total": 3, "pageLimit": 1, "pageOffset": 1, "pageIds": [2], "emptyStatus": "missing",
+    "sumAmount": "20.00", "statusCounts": {"paid": 2, "open": 1}
+  }
+}