RHIDP-18071: extend environment variable parsing behaviour

docs/user_doc/deployment_guide.md

@@ -4,6 +4,8 @@
 
 * [Preface](#preface)
 * [Deployment methods](#deployment-methods)
+* [Configuration modes](#configuration-modes)
+    * [Environment variable references](#environment-variable-references)
 * [Integration with OGX framework](#integration-with-ogx-framework)
     * [OGX as a library](#ogx-as-a-library)
     * [OGX as a server](#ogx-as-a-server)
@@ -108,6 +110,38 @@ fails if a unified synthesis input and `library_client_config_path` are both
 present.
 
 
+### Environment variable references
+
+`lightspeed-stack.yaml` accepts `${VAR}` wherever `${env.VAR}` is supported.
+Both forms behave the same. Variable names contain one or more uppercase
+letters, digits, or underscores.
+
+Both spellings support these forms:
+
+- `${VAR}` or `${env.VAR}` uses the variable's value. Resolution fails if the
+  variable is missing or empty.
+- `${VAR:=default}` or `${env.VAR:=default}` uses `default` when the variable
+  is missing or empty.
+- `${VAR:+value}` or `${env.VAR:+value}` uses `value` when the variable is
+  non-empty. Otherwise, it substitutes an empty string.
+
+The shell-style `:-` default operator is not supported. OGX converts resolved
+numeric strings to numbers and boolean strings to booleans. An empty result
+becomes `null`.
+
+Both references below use defaults if their variables are missing or empty:
+
+```yaml
+service:
+  host: "${SERVICE_HOST:=0.0.0.0}"
+  port: "${env.SERVICE_PORT:=8080}"
+```
+
+LCORE resolves its configuration values while loading. For deferred values,
+including those in `ogx.config.native_override`, LCORE writes `${env.VAR}`
+references to the generated `run.yaml`. OGX resolves them at startup.
+
+
 
 ## Integration with OGX framework
 
@@ -283,7 +317,7 @@ llama_stack:
    ```
 
 3. Replace literal secrets. If your `run.yaml` contained secret values
-   directly, replace them with `${env.MY_VAR}` environment references —
+   directly, replace them with `${MY_VAR}` or `${env.MY_VAR}` environment references —
    the migrated file otherwise carries them onto disk verbatim (the
    synthesized output is written owner-only, mode 0600, as a safety net).
 

src/configuration.py

@@ -4,10 +4,6 @@
 
 import yaml
 
-# We want to support environment variable replacement in the configuration
-# similarly to how it is done in OGX, so we use their function directly
-from ogx.core.stack import replace_env_vars
-
 import constants
 from cache.cache import Cache
 from cache.cache_factory import CacheFactory
@@ -40,25 +36,26 @@
 from quota.quota_limiter import QuotaLimiter
 from quota.quota_limiter_factory import QuotaLimiterFactory
 from quota.token_usage_history import TokenUsageHistory
+from utils.env_vars import normalize_env_references, resolve_env_references
 
 logger = get_logger(__name__)
 
 
 def replace_env_vars_preserving_native_override(
     config_dict: dict[Any, Any],
 ) -> dict[Any, Any]:
-    """Resolve ${env.*} references in the config, except in native_override.
+    """Resolve ${env.VAR} and ${VAR} references, except in native_override.
 
     LCORE resolves environment-variable references throughout
     lightspeed-stack.yaml so typed fields receive concrete values. But
     ``ogx.config.native_override`` is raw OGX schema that OGX
     resolves itself, in memory, at its own startup. Resolving it eagerly
-    here would (a) defeat the ${env.*} pattern LCORE recommends for secrets and
+    here would (a) defeat the ${env.VAR} and ${VAR} patterns for secrets and
     (b) pull resolved secrets into the loaded Configuration model, which is
     logged at startup. So native_override is held aside, the rest of the config
-    is resolved, and the raw (unresolved) native_override is restored verbatim.
-    Synthesis reads native_override from the raw YAML, so this does not change
-    the generated run.yaml.
+    is resolved, and native_override is restored with ${VAR} normalized to
+    ${env.VAR} without resolving it. Synthesis reads native_override from the
+    raw YAML and applies the same normalization to the generated run.yaml.
 
     Parameters:
         config_dict: The parsed lightspeed-stack.yaml.
@@ -69,24 +66,24 @@ def replace_env_vars_preserving_native_override(
 
     """
     if not isinstance(config_dict, dict):
-        return replace_env_vars(config_dict)
+        return resolve_env_references(config_dict)
 
     ogx_section = config_dict.get("ogx")
     if ogx_section is None:
         ogx_section = config_dict.get("llama_stack")
     ogx_config = ogx_section.get("config") if isinstance(ogx_section, dict) else None
     if not (isinstance(ogx_config, dict) and "native_override" in ogx_config):
-        return replace_env_vars(config_dict)
+        return resolve_env_references(config_dict)
 
     raw_override = ogx_config["native_override"]
     ogx_config["native_override"] = {}  # keep secrets out of env resolution
-    resolved = replace_env_vars(config_dict)
+    resolved = resolve_env_references(config_dict)
     ogx_config["native_override"] = raw_override  # restore source dict if reused
     resolved_ogx = (resolved.get("ogx") or resolved.get("llama_stack") or {}).get(
         "config"
     )
     if isinstance(resolved_ogx, dict):
-        resolved_ogx["native_override"] = raw_override
+        resolved_ogx["native_override"] = normalize_env_references(raw_override)
     return resolved
 
 

src/ogx_configuration.py

@@ -26,11 +26,11 @@
 from urllib.parse import urljoin
 
 import yaml
-from ogx.core.stack import replace_env_vars
 from pydantic import SecretStr
 
 import constants
 from log import get_logger, setup_logging
+from utils.env_vars import normalize_env_references, resolve_env_references
 
 logger = get_logger(__name__)
 
@@ -315,10 +315,10 @@ def construct_vector_stores_section(
             output = ogx_config["registered_resources"]["vector_stores"].copy()
 
     # append new vector_stores entries, skipping duplicates
-    # Resolve ${env.VAR} patterns so comparisons work when existing entries
+    # Resolve ${env.VAR} and ${VAR} so comparisons work when existing entries
     # use environment variable references and new entries have resolved values.
     existing_store_ids = {
-        replace_env_vars(vs.get("vector_store_id", "")) for vs in output
+        resolve_env_references(vs.get("vector_store_id", "")) for vs in output
     }
     added = 0
     for brag in byok_rag:
@@ -879,8 +879,8 @@ def enrich_solr(  # pylint: disable=too-many-locals,too-many-statements,too-many
     base_url_raw = (
         str(rhokp_raw) if rhokp_raw is not None else constants.RH_SERVER_OKP_DEFAULT_URL
     )
-    # Resolve environment variables in the URL (e.g., ${env.RH_SERVER_OKP})
-    base_url = replace_env_vars(base_url_raw)
+    # Resolve ${env.RH_SERVER_OKP} and ${RH_SERVER_OKP} before joining the URL path.
+    base_url = resolve_env_references(base_url_raw)
     solr_url = urljoin(base_url, "/solr")
 
     logger.info("Enriching OGX config with OKP")
@@ -1449,6 +1449,8 @@ def synthesize_configuration(  # pylint: disable=too-many-locals
         dict[str, Any]: The synthesized OGX configuration.
 
     """
+    # Keep raw references deferred for OGX startup while canonicalizing their syntax.
+    lcs_config = normalize_env_references(lcs_config)
     unified = ogx_config_section(lcs_config).get("config")
 
     # 1-2. Select the baseline.
@@ -1682,6 +1684,9 @@ def generate_configuration(
     """
     logger.info("Reading OGX configuration from file %s", input_file)
 
+    # Normalize raw LCORE enrichment references without looking up their values.
+    config = normalize_env_references(config)
+
     with open(input_file, encoding="utf-8") as file:
         ogx_config = yaml.safe_load(file)
 

src/utils/env_vars.py

@@ -0,0 +1,69 @@
+"""Normalize environment references and resolve them using OGX semantics."""
+
+import re
+from typing import Any, Final
+
+from ogx.core.stack import replace_env_vars
+
+# Match ${VAR} and ${env.VAR}, optionally with :=default or :+value.
+# Names use OGX's uppercase/digit/underscore grammar; operands end at the first }.
+_ENV_REFERENCE_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"\$\{(?:env\.)?[A-Z0-9_]+(?::[=+][^}]*)?\}"
+)
+
+
+def canonicalize_reference(match: re.Match[str]) -> str:
+    """Preserve OGX references and prefix short references with env.
+
+    Parameters:
+        match: A supported reference found by the normalizer's regex.
+
+    Returns:
+        The reference in canonical ${env.VAR} syntax, without resolving it.
+
+    """
+    # group(0) is the complete matched reference, including ${ and }.
+    reference = match.group(0)
+    if reference.startswith("${env."):
+        return reference
+    return reference.replace("${", "${env.", 1)
+
+
+def normalize_env_references(config: Any) -> Any:
+    """Convert supported ${VAR} references to ${env.VAR} without resolving them.
+
+    Recursively visit dictionary values and list elements, rebuilding each
+    container and normalizing its strings. Keys and existing OGX references are
+    unchanged. Variable names and operators follow OGX's resolver grammar.
+
+    Parameters:
+        config: A configuration value, including nested containers and scalars.
+
+    Returns:
+        The configuration with supported short references canonicalized.
+
+    """
+    if isinstance(config, str):
+        # sub calls the canonicalizer for each match, preserving surrounding text.
+        # For ${HOST}:${PORT}, each reference gets its own canonical replacement.
+        return _ENV_REFERENCE_PATTERN.sub(canonicalize_reference, config)
+    if isinstance(config, dict):
+        return {key: normalize_env_references(value) for key, value in config.items()}
+    if isinstance(config, list):
+        return [normalize_env_references(value) for value in config]
+    return config
+
+
+def resolve_env_references(config: Any) -> Any:
+    """Normalize environment references, then delegate their resolution to OGX.
+
+    Parameters:
+        config: A configuration value, including nested containers and scalars.
+
+    Returns:
+        The recursively resolved configuration. OGX may convert substituted
+        strings to numbers, booleans, or None; other scalar values pass through.
+
+    """
+    normalized_config = normalize_env_references(config)
+    return replace_env_vars(normalized_config)

tests/conftest.py

@@ -0,0 +1,11 @@
+"""Fixtures shared by the unit and integration test suites."""
+
+import pytest
+
+
+@pytest.fixture(
+    name="reference_prefix", params=["", "env."], ids=["short", "canonical"]
+)
+def reference_prefix_fixture(request: pytest.FixtureRequest) -> str:
+    """Exercise both environment-reference spellings."""
+    return str(request.param)

tests/integration/test_unified_synthesis.py

@@ -602,3 +602,126 @@ def test_synthesized_config_output_override(
     assert Path(written) == custom_output
     assert isinstance(yaml.safe_load(custom_output.read_text(encoding="utf-8")), dict)
     assert not (tmp_path / constants.DEFAULT_SYNTHESIZED_CONFIG_PATH).exists()
+
+
+@pytest.mark.parametrize("secret_is_set", [True, False], ids=["set", "unset"])
+def test_synthesis_defers_native_override_environment_references(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    reference_prefix: str,
+    secret_is_set: bool,
+) -> None:
+    """Emit deferred references with secrets either set or absent during synthesis."""
+    secret = "native-override-secret-must-stay-off-disk"
+    monkeypatch.delenv("AZURE_API_KEY", raising=False)
+    if secret_is_set:
+        monkeypatch.setenv("AZURE_API_KEY", secret)
+    override = copy.deepcopy(_OPERATOR_RUN_YAML)
+    override["providers"]["inference"][0]["config"][
+        "api_key"
+    ] = f"${{{reference_prefix}AZURE_API_KEY}}"
+    config = _base_config_dict()
+    config["ogx"] = {
+        "use_as_library_client": True,
+        "config": {"baseline": "empty", "native_override": override},
+    }
+
+    synthesized, output = _load_and_synthesize(tmp_path, config)
+
+    provider = synthesized["providers"]["inference"][0]
+    assert provider["config"]["api_key"] == "${env.AZURE_API_KEY}"
+    assert secret not in output.read_text(encoding="utf-8")
+
+
+def test_synthesis_defers_inference_environment_references(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, reference_prefix: str
+) -> None:
+    """Keep explicit API key and URL references canonical in generated config."""
+    secret = "inference-secret-must-stay-off-disk"
+    base_url = "https://inference.example.test"
+    monkeypatch.setenv("LCORE_TEST_API_KEY", secret)
+    monkeypatch.setenv("LCORE_TEST_INFERENCE_URL", base_url)
+    config = _base_config_dict()
+    config["ogx"] = {"use_as_library_client": True, "config": {"baseline": "empty"}}
+    config["inference"] = {
+        "providers": [
+            {
+                "type": "openai",
+                "extra": {
+                    "api_key": f"${{{reference_prefix}LCORE_TEST_API_KEY}}",
+                    "base_url": f"${{{reference_prefix}LCORE_TEST_INFERENCE_URL}}",
+                },
+            }
+        ]
+    }
+
+    synthesized, output = _load_and_synthesize(tmp_path, config)
+
+    provider_config = synthesized["providers"]["inference"][0]["config"]
+    assert provider_config["api_key"] == "${env.LCORE_TEST_API_KEY}"
+    assert provider_config["base_url"] == "${env.LCORE_TEST_INFERENCE_URL}"
+    text = output.read_text(encoding="utf-8")
+    assert secret not in text
+    assert base_url not in text
+
+
+@pytest.mark.parametrize("source", ["byok", "vector_store"])
+def test_synthesis_defers_pgvector_environment_references(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    reference_prefix: str,
+    source: str,
+) -> None:
+    """Preserve canonical connection references through both pgvector enrichments."""
+    variables = {
+        "host": ("LCORE_TEST_PG_HOST", "postgres.example.test"),
+        "port": ("LCORE_TEST_PG_PORT", "5432"),
+        "db": ("LCORE_TEST_PG_DATABASE", "knowledge"),
+        "user": ("LCORE_TEST_PG_USER", "lightspeed"),
+        "password": ("LCORE_TEST_PG_PASSWORD", "pg-secret-must-stay-off-disk"),
+    }
+    for variable, value in variables.values():
+        monkeypatch.setenv(variable, value)
+    references = {
+        field: f"${{{reference_prefix}{variable}}}"
+        for field, (variable, _) in variables.items()
+    }
+    config = _base_config_dict()
+    config["ogx"] = {"use_as_library_client": True, "config": {"baseline": "empty"}}
+    if source == "byok":
+        config.setdefault("rag", {})["byok"] = {
+            "stores": [
+                {
+                    "rag_id": "pg",
+                    "backend": "pgvector",
+                    "vector_db_id": "knowledge-store",
+                    "embedding_model": "test-model",
+                    "embedding_dimension": 768,
+                    **references,
+                }
+            ]
+        }
+    else:
+        config["vector_store"] = {
+            "default_provider": "pg",
+            "providers": [
+                {
+                    "id": "pg",
+                    "type": "pgvector",
+                    "embedding_model": "test-model",
+                    "embedding_dimension": 768,
+                    "config": references,
+                }
+            ],
+        }
+
+    synthesized, output = _load_and_synthesize(tmp_path, config)
+
+    provider = next(
+        provider
+        for provider in synthesized["providers"]["vector_io"]
+        if provider["provider_type"] == "remote::pgvector"
+    )
+    for field, (variable, _) in variables.items():
+        assert provider["config"][field] == f"${{env.{variable}}}"
+    assert variables["password"][1] not in output.read_text(encoding="utf-8")