Support current Ossie profiles and preserve source semantics

pyproject.toml

@@ -199,7 +199,7 @@ force-exclude = true
 extend-exclude = [
     "sidemantic/adapters/malloy_grammar",  # ANTLR-generated files
     "sidemantic/adapters/holistics_grammar",  # ANTLR-generated files
-    "tests/ossie-fixtures/upstream/validation/validate.py",  # Exact pinned Apache validator fixture
+    "tests/ossie-fixtures/upstream/**/validation/validate.py",  # Exact pinned Apache validator fixtures
 ]
 
 [tool.ruff.lint]

sidemantic/adapters/ossie.py

@@ -32,6 +32,7 @@
     sort_diagnostics,
     synthesize_ossie_document,
 )
+from sidemantic.interchange.ossie.profiles import CURRENT_OSSIE_SCHEMA_COMMIT
 
 _GENERATED_DIRECTORIES = frozenset({"dbt_packages", "target"})
 
@@ -75,6 +76,7 @@ def __init__(
         export_scope_name: str | None = None,
         expression_dialect: str | None = None,
         schema_version: str = "0.2.0.dev0",
+        schema_revision: str | None = None,
         serialization: OssieSerialization | str | None = None,
     ) -> None:
         if not target_dialect.strip():
@@ -84,8 +86,10 @@ def __init__(
         self._export_scope_name = export_scope_name
         self._expression_dialect = expression_dialect
         self._schema_version = schema_version
+        self._schema_revision = schema_revision
         self._serialization = OssieSerialization(serialization) if serialization is not None else None
         self._parse_options = OssieParseOptions(
+            schema_revision=schema_revision,
             consumer_profile=consumer_profile,
             import_policy=import_policy,
             source_dialect=source_dialect,
@@ -212,6 +216,7 @@ def export(
         scope_name: str | None = None,
         expression_dialect: str | None = None,
         schema_version: str | None = None,
+        schema_revision: str | None = None,
         serialization: OssieSerialization | str | None = None,
         portable_only: bool = False,
     ) -> None:
@@ -234,6 +239,7 @@ def export(
             scope_name=selected_scope,
             expression_dialect=selected_dialect,
             schema_version=schema_version or self._schema_version,
+            schema_revision=schema_revision or self._schema_revision or CURRENT_OSSIE_SCHEMA_COMMIT,
             serialization=output_serialization,
             consumer_profile=self._parse_options.consumer_profile,
             portable_only=portable_only,

sidemantic/cli.py

@@ -1540,6 +1540,11 @@ def convert(
         "--ossie-schema-version",
         help="Explicit pinned Ossie schema version for synthesized output",
     ),
+    ossie_schema_revision: str = typer.Option(
+        None,
+        "--ossie-schema-revision",
+        help="Pinned Ossie draft revision for synthesized output (commit SHA)",
+    ),
     ossie_consumer_profile: str = typer.Option(
         None,
         "--ossie-consumer-profile",
@@ -1603,7 +1608,7 @@ def convert(
             from sidemantic.fidelity import capture_import_report
 
             source_adapter_options = None
-            if source_format != "auto" and get_semantic_format(source_format, operation="import").name == "ossie":
+            if source_format == "auto" or get_semantic_format(source_format, operation="import").name == "ossie":
                 source_adapter_options = {}
                 if ossie_scope is not None:
                     source_adapter_options["scope_id"] = ossie_scope
@@ -1632,6 +1637,8 @@ def convert(
                     target_adapter_options = {"consumer_profile": ossie_consumer_profile}
                 if ossie_schema_version is not None:
                     target_export_options["schema_version"] = ossie_schema_version
+                if ossie_schema_revision is not None:
+                    target_export_options["schema_revision"] = ossie_schema_revision
 
             with progress(f"Converting semantic definitions to {target_format}"):
                 with capture_import_report() as fidelity_report:

sidemantic/formats.py

@@ -261,23 +261,22 @@ def load_semantic_source(
 
     File inputs are always exact: auto-discovery parses only the named file and
     never scans its siblings. Directory inputs retain the existing project-wide
-    discovery behavior.
+    discovery behavior. In auto mode, adapter options apply only to detected
+    Ossie sources; other format options require an explicit source format.
     """
     source_path = Path(source)
     if not source_path.exists():
         raise FileNotFoundError(f"Semantic source does not exist: {source_path}")
 
     if source_format.strip().lower() == "auto":
-        if adapter_options:
-            raise ValueError("adapter_options require an explicit source_format")
         from sidemantic.core.semantic_layer import SemanticLayer
         from sidemantic.loaders import load_from_directory, load_from_file
 
         layer = SemanticLayer()
         if source_path.is_file():
-            load_from_file(layer, source_path)
+            load_from_file(layer, source_path, ossie_adapter_options=adapter_options)
         else:
-            load_from_directory(layer, source_path)
+            load_from_directory(layer, source_path, ossie_adapter_options=adapter_options)
         return layer.graph
 
     spec = get_semantic_format(source_format, operation="import")

sidemantic/interchange/ossie/__init__.py

@@ -18,6 +18,8 @@
     OssieOntologyDocument,
     UnsupportedOssieDocument,
     freeze_json,
+    is_logical_document_data,
+    logical_model_entries,
     thaw_json,
 )
 from sidemantic.interchange.ossie.lowering import OssieLoweringResult, lower_ossie_document
@@ -27,9 +29,12 @@
     parse_ossie_document,
 )
 from sidemantic.interchange.ossie.profiles import (
+    CURRENT_OSSIE_SCHEMA_COMMIT,
     DBT_1_12_0_1_0_ALIAS,
     DBT_1_12_0_1_1,
+    LEGACY_OSSIE_SCHEMA_COMMIT,
     OSSIE_CORE_0_1_1,
+    OSSIE_CORE_0_2_0_CURRENT,
     OSSIE_CORE_0_2_0_DEV0,
     OSSIE_PROFILES,
     OssieConsumerProfile,
@@ -59,10 +64,13 @@
 )
 
 __all__ = [
+    "CURRENT_OSSIE_SCHEMA_COMMIT",
+    "LEGACY_OSSIE_SCHEMA_COMMIT",
     "DBT_1_12_0_1_0_ALIAS",
     "DBT_1_12_0_1_1",
     "OSSIE_CORE_0_1_1",
     "OSSIE_CORE_0_2_0_DEV0",
+    "OSSIE_CORE_0_2_0_CURRENT",
     "OSSIE_PROFILES",
     "FrozenJSONValue",
     "FrozenJSONObject",
@@ -94,6 +102,8 @@
     "UnsupportedOssieDocument",
     "diagnostic_sort_key",
     "freeze_json",
+    "is_logical_document_data",
+    "logical_model_entries",
     "lower_ossie_document",
     "parse_ossie_document",
     "resolve_ossie_profile",

sidemantic/interchange/ossie/documents.py

@@ -19,6 +19,29 @@
 ParsedJSONValue: TypeAlias = JSONScalar | Mapping[str, object] | list[object] | tuple[object, ...]
 
 
+def is_logical_document_data(data: Mapping[str, object]) -> bool:
+    """Recognize current flat models and the earlier model-array envelope."""
+
+    return (
+        "semantic_model" in data
+        or "datasets" in data
+        or ("name" in data and "ontology" not in data and "ontology_mappings" not in data)
+    )
+
+
+def logical_model_entries(data: Mapping[str, object]) -> tuple[tuple[str, object], ...]:
+    """Expose model values and their source pointers without rewriting source data."""
+
+    if "semantic_model" in data:
+        models = data["semantic_model"]
+        if isinstance(models, (list, tuple)):
+            return tuple((f"/semantic_model/{index}", model) for index, model in enumerate(models))
+        return ()
+    if is_logical_document_data(data):
+        return (("", data),)
+    return ()
+
+
 @dataclass(frozen=True, slots=True)
 class FrozenJSONObject(Mapping[str, "FrozenJSONValue"]):
     """An insertion-ordered, deeply immutable JSON object."""
@@ -129,6 +152,7 @@ class _OssieDocumentBase:
     canonical_data: ParsedJSONValue | FrozenJSONObject
     serialization: OssieSerialization
     source: OssieDocumentSource | None = None
+    schema_revision: str | None = None
 
     _known_root_fields: ClassVar[frozenset[str]] = frozenset()
 
@@ -170,7 +194,21 @@ def to_parsed_data(self) -> object:
 class OssieLogicalDocument(_OssieDocumentBase):
     """A logical-layer Ossie document, before validation or runtime lowering."""
 
-    _known_root_fields: ClassVar[frozenset[str]] = frozenset({"version", "dialects", "vendors", "semantic_model"})
+    _known_root_fields: ClassVar[frozenset[str]] = frozenset(
+        {
+            "version",
+            "dialects",
+            "vendors",
+            "semantic_model",
+            "name",
+            "description",
+            "ai_context",
+            "datasets",
+            "relationships",
+            "metrics",
+            "custom_extensions",
+        }
+    )
 
     def __post_init__(self) -> None:
         super(OssieLogicalDocument, self).__post_init__()
@@ -183,16 +221,19 @@ def semantic_model_value(self) -> FrozenJSONValue | None:
 
     @property
     def semantic_models(self) -> tuple[FrozenJSONValue, ...]:
-        value = self.semantic_model_value
-        return value if isinstance(value, tuple) else ()
+        return tuple(model for _, model in self.semantic_model_entries)
+
+    @property
+    def semantic_model_entries(self) -> tuple[tuple[str, FrozenJSONValue], ...]:
+        return logical_model_entries(self.canonical_data)
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
 class OssieOntologyDocument(_OssieDocumentBase):
     """An ontology-layer Ossie document, preserved without reasoning semantics."""
 
     _known_root_fields: ClassVar[frozenset[str]] = frozenset(
-        {"version", "name", "description", "ai_context", "ontology", "ontology_mappings"}
+        {"version", "name", "description", "ai_context", "requires", "ontology", "ontology_mappings", "prefixes"}
     )
 
     def __post_init__(self) -> None:

sidemantic/interchange/ossie/expression_validation.py

@@ -33,7 +33,7 @@
 )
 
 
-def scalar_sql_expression_error(expression: str, *, sqlglot_dialect: str | None) -> str | None:
+def scalar_sql_expression_error(expression: str, *, sqlglot_dialect: str | None, row_level: bool = False) -> str | None:
     """Return why SQL is not one scalar Ossie expression, otherwise ``None``.
 
     Parsing uses the selected executable dialect. The structural gate rejects
@@ -49,7 +49,11 @@ def scalar_sql_expression_error(expression: str, *, sqlglot_dialect: str | None)
         return "expression must contain exactly one SQL expression"
 
     root = parsed[0]
+    if isinstance(root, (exp.Alias, exp.Aliases, exp.Star)):
+        return "expression must be a scalar value, without a projection alias or wildcard"
     for node in root.walk():
         if isinstance(node, _FORBIDDEN_NODE_TYPES):
             return f"Ossie expressions cannot contain {type(node).__name__}"
+        if row_level and isinstance(node, exp.AggFunc) and node.find_ancestor(exp.Window) is None:
+            return "Ossie dataset fields are row-level expressions and cannot contain aggregates"
     return None

sidemantic/interchange/ossie/identifier.py

@@ -5,6 +5,21 @@
 OSSIE_IDENTIFIER_MAX_LENGTH = 128
 
 
+def identifier_syntax_valid(identifier: str) -> bool:
+    """Accept regular names or nonempty ANSI delimited names with escaped quotes."""
+
+    if identifier.startswith('"'):
+        if len(identifier) < 3 or not identifier.endswith('"'):
+            return False
+        body = identifier[1:-1]
+        return "\x00" not in body and '"' not in body.replace('""', "")
+    return (
+        bool(identifier)
+        and (identifier[0].isalpha() or identifier[0] == "_")
+        and all(character.isalnum() or character == "_" for character in identifier)
+    )
+
+
 def is_quoted_identifier(identifier: str) -> bool:
     """Return whether *identifier* uses Ossie's ANSI double-quote form."""
 

sidemantic/interchange/ossie/lowering.py

@@ -24,7 +24,7 @@
     OssieSourceLocation,
     sort_diagnostics,
 )
-from sidemantic.interchange.ossie.documents import OssieLogicalDocument, OssieOntologyDocument
+from sidemantic.interchange.ossie.documents import OssieLogicalDocument, OssieOntologyDocument, logical_model_entries
 from sidemantic.interchange.ossie.expression_validation import scalar_sql_expression_error
 from sidemantic.interchange.ossie.identifier import identifier_within_limit, normalize_identifier
 from sidemantic.interchange.ossie.parser import OssieParseResult
@@ -178,10 +178,15 @@ def _sql_expression_error(expression: str, target_dialect: str) -> str | None:
 def _classify_source(source: str, source_dialect: str | None) -> tuple[str, str] | None:
     dialect = _SQLGLOT_DIALECTS.get(_normalize_dialect(source_dialect)) if source_dialect else None
     try:
-        parsed = sqlglot.parse_one(source, read=dialect)
+        statements = sqlglot.parse(source, read=dialect)
+        if len(statements) != 1:
+            return None
+        parsed = statements[0]
     except sqlglot.errors.ParseError:
         parsed = None
     if isinstance(parsed, (exp.Query, exp.Subquery)):
+        if any(not select.expressions for select in parsed.find_all(exp.Select)):
+            return None
         return "query", source
 
     try:
@@ -256,6 +261,7 @@ def _lower_scope(
     *,
     scope_id: str,
     scope_index: int,
+    scope_pointer: str,
     document_id: str,
     target_dialect: str,
     diagnostics: list[OssieDiagnostic],
@@ -272,7 +278,7 @@ def _lower_scope(
     registration_token = set_current_layer(None)
     try:
         for dataset_index, dataset, dataset_name in _unique_named_items(dataset_values):
-            pointer = f"/semantic_model/{scope_index}/datasets/{dataset_index}"
+            pointer = f"{scope_pointer}/datasets/{dataset_index}"
             source = dataset.get("source")
             if not isinstance(source, str) or not source.strip():
                 diagnostics.append(
@@ -400,7 +406,7 @@ def _lower_scope(
 
         relationships = _array(semantic_model.get("relationships")) if runtime_override is None else ()
         for relationship_index, relationship, edge_id in _unique_named_items(relationships):
-            pointer = f"/semantic_model/{scope_index}/relationships/{relationship_index}"
+            pointer = f"{scope_pointer}/relationships/{relationship_index}"
             from_name = _name(relationship.get("from"))
             to_name = _name(relationship.get("to"))
             from_columns = _array(relationship.get("from_columns"))
@@ -463,7 +469,7 @@ def _lower_scope(
 
         metrics = _array(semantic_model.get("metrics")) if runtime_override is None else ()
         for metric_index, metric, metric_name in _unique_named_items(metrics):
-            pointer = f"/semantic_model/{scope_index}/metrics/{metric_index}"
+            pointer = f"{scope_pointer}/metrics/{metric_index}"
             selected = _expression_for_target(metric.get("expression"), target_dialect)
             if selected is None:
                 diagnostics.append(
@@ -618,19 +624,16 @@ def lower_ossie_document(
             lowering_diagnostics=tuple(diagnostics),
         )
 
-    parsed = document.to_parsed_data()
-    root = _mapping(parsed)
-    semantic_models = _array(root.get("semantic_model")) if root else None
     named_models = [
-        (index, model, model_name)
-        for index, value in enumerate(semantic_models or ())
+        (index, pointer, model, model_name)
+        for index, (pointer, value) in enumerate(logical_model_entries(document.to_parsed_data()))
         if (model := _mapping(value)) is not None
         if (model_name := _name(model.get("name"))) is not None
     ]
-    name_counts = Counter(normalize_identifier(name) for _, _, name in named_models if identifier_within_limit(name))
+    name_counts = Counter(normalize_identifier(name) for _, _, _, name in named_models if identifier_within_limit(name))
     document_id = _document_id(parse_result)
     scopes = []
-    for index, semantic_model, name in named_models:
+    for index, scope_pointer, semantic_model, name in named_models:
         if not identifier_within_limit(name):
             continue
         scope_id = name if name_counts[normalize_identifier(name)] == 1 else f"{name}@{index}"
@@ -644,7 +647,7 @@ def lower_ossie_document(
                     parse_result,
                     code="ossie.lowering.runtime_extension_invalid",
                     message=f"Sidemantic runtime extension cannot be restored safely: {exc}",
-                    pointer=f"/semantic_model/{index}/custom_extensions",
+                    pointer=f"{scope_pointer}/custom_extensions",
                     scope=scope_id,
                 )
             )
@@ -657,7 +660,7 @@ def lower_ossie_document(
                     severity=OssieDiagnosticSeverity.WARNING,
                     code="ossie.lowering.runtime_extension_restored",
                     message="Restored native Sidemantic runtime semantics; other consumers require Sidemantic extension support.",
-                    json_pointer=f"/semantic_model/{index}/custom_extensions",
+                    json_pointer=f"{scope_pointer}/custom_extensions",
                     scope=scope_id,
                     source=_source_location(parse_result),
                     profile=parse_result.profile,
@@ -669,6 +672,7 @@ def lower_ossie_document(
                 semantic_model,
                 scope_id=scope_id,
                 scope_index=index,
+                scope_pointer=scope_pointer,
                 document_id=document_id,
                 target_dialect=selected_target,
                 diagnostics=diagnostics,