fix(TC-500): preflight sealed Node startup configuration

.github/workflows/docker.yml

@@ -167,6 +167,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: [version-guard]
     if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    outputs:
+      digest: ${{ steps.build.outputs.digest }}
     permissions:
       contents: read
       packages: write
@@ -218,6 +220,7 @@ jobs:
             type=raw,value=dstack${{ steps.build_features.outputs.image_suffix }},enable={{is_default_branch}}
 
       - name: Build and push dstack Docker image
+        id: build
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -230,11 +233,21 @@ jobs:
 
   deploy-phala:
     runs-on: ubuntu-latest
-    needs: [build, build-dstack]
+    needs: [version-guard, build, build-dstack]
     if: github.event_name == 'release' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.deploy_phala)
+    permissions:
+      contents: read
+      packages: read
     steps:
       - uses: actions/checkout@v4
 
+      - name: Log in to GHCR for runtime configuration preflight
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
@@ -248,35 +261,67 @@ jobs:
       - name: Install Phala CLI
         run: npm install -g phala@1.1.19
 
-      - name: Update compose with release tag
+      - name: Validate and pin deployment image
+        env:
+          BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }}
         run: |
-          if [ "${{ github.event_name }}" = "release" ]; then
-            TAG="${{ github.event.release.tag_name }}"
-          else
-            TAG="${{ inputs.image_version }}"
-            if [ -z "${TAG}" ]; then
-              if [ "${GITHUB_REF_TYPE}" != "tag" ]; then
-                echo "::error::workflow_dispatch deploy_phala requires image_version unless the workflow is run from a tag"
-                exit 1
-              fi
-              TAG="${GITHUB_REF_NAME}"
-            fi
+          set -euo pipefail
+          DIGEST="${BUILD_DIGEST}"
+          if ! [[ "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "::error::build-dstack did not produce an immutable OCI digest"
+            exit 1
           fi
-          # metadata-action emits {{version}} without the leading v (e.g. 1.3.0).
-          # Strip a leading v from the tag to match the pushed image tag.
-          VERSION="${TAG#v}"
-          DUCKDB_SUFFIX=""
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then
-            DUCKDB_SUFFIX="-duckdb"
+          export RESOLVED_IMAGE="${REGISTRY}/${IMAGE_NAME}@${DIGEST}"
+          docker pull "${RESOLVED_IMAGE}"
+          REVISION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
+          if [ "${REVISION}" != "${GITHUB_SHA}" ]; then
+            echo "::error::candidate OCI revision '${REVISION}' does not match workflow commit '${GITHUB_SHA}'"
+            exit 1
           fi
-          # The prod CVM uses the dstack-suffixed image. Replace the floating
-          # ":dstack" tag in the checked-in compose with the versioned tag
-          # built by the build-dstack job (e.g. ":1.3.0-dstack" or
-          # ":1.3.0-dstack-duckdb").
-          sed -i "s|ghcr.io/tinycloudlabs/tinycloud-node:dstack|ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}|g" docker-compose.dstack-postgres.yaml
-          echo "Resolved image tag: ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}"
+
+          # Keep this immutable reference through preflight, compose parsing,
+          # and Phala. A version tag can be retargeted between build and deploy.
+          python3 - <<'PY'
+          from pathlib import Path
+          import os
+
+          path = Path("docker-compose.dstack-postgres.yaml")
+          content = path.read_text()
+          floating = "ghcr.io/tinycloudlabs/tinycloud-node:dstack"
+          if content.count(floating) != 1:
+              raise SystemExit("expected exactly one floating dstack image in compose")
+          path.write_text(content.replace(floating, os.environ["RESOLVED_IMAGE"]))
+          PY
+          echo "Resolved immutable candidate: ${RESOLVED_IMAGE}"
           cat docker-compose.dstack-postgres.yaml
 
+      # TC-500: Phala considers a CVM ready before compose services have
+      # started. Validate the production-sealed bundle using the exact image
+      # selected above against the production database/TLS/key configuration,
+      # without opening a database, resolving dstack, network, or socket
+      # access, before the deploy step can replace the healthy CVM.
+      - name: Preflight sealed runtime configuration
+        env:
+          TINYCLOUD_STORAGE__DATABASE: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }}
+          TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: ${{ secrets.PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64 }}
+          TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full
+          TINYCLOUD_KEYS__TYPE: Dstack
+        run: |
+          set -euo pipefail
+          IMAGE="$(sed -n -E 's/^    image: (ghcr.io\/tinycloudlabs\/tinycloud-node(:[^[:space:]]+|@sha256:[0-9a-f]{64}))$/\1/p' docker-compose.dstack-postgres.yaml)"
+          if [ "$(printf '%s\n' "${IMAGE}" | awk 'NF { count += 1 } END { print count + 0 }')" != "1" ]; then
+            echo "::error::Expected exactly one resolved TinyCloud Node image for runtime preflight"
+            exit 1
+          fi
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --security-opt no-new-privileges \
+            -e TINYCLOUD_STORAGE__DATABASE \
+            -e TINYCLOUD_SHARE_EMAIL__ENABLED=true \
+            -e TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64 \
+            -e TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE \
+            -e TINYCLOUD_KEYS__TYPE \
+            "${IMAGE}" --validate-config
+
       - name: Deploy to Phala Cloud
         env:
           PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }}
@@ -408,6 +453,7 @@ jobs:
             /policy/v3/policies
             /policy/v3/challenges
             /policy/v3/delegations
+            /policy/v3/deliveries/authorize
           )
 
           # The direct TEE ingress obtains/loads its certificate after the
@@ -435,8 +481,20 @@ jobs:
               --data '{}' \
               "${NODE_ORIGIN}${route}")"
 
-            if [ "${STATUS}" != "400" ] && [ "${STATUS}" != "422" ]; then
-              echo "::error::${route} returned HTTP ${STATUS}; expected 400 or 422 for an intentionally invalid request. The Policy/v3 production contract is not mounted correctly."
+            # Delivery is authenticated before request-body validation, so an
+            # intentionally unsigned body is expected to return 401. Other
+            # control-plane routes reject the body first. Neither case may
+            # silently accept a missing route or a server failure.
+            if [ "${route}" = "/policy/v3/deliveries/authorize" ]; then
+              EXPECTED="401"
+              case "${STATUS}" in 401) VALID=true ;; *) VALID=false ;; esac
+            else
+              EXPECTED="400 or 422"
+              case "${STATUS}" in 400|422) VALID=true ;; *) VALID=false ;; esac
+            fi
+
+            if [ "${VALID}" != true ]; then
+              echo "::error::${route} returned HTTP ${STATUS}; expected ${EXPECTED} for an intentionally invalid request. The Policy/v3 production contract is not mounted correctly."
               head -c 1000 "${RESPONSE_FILE}" || true
               rm -f "${RESPONSE_FILE}"
               exit 1

scripts/check-deployment-policy-probes.mjs

@@ -5,9 +5,11 @@ const expectedRoutes = [
   "/policy/v3/policies",
   "/policy/v3/challenges",
   "/policy/v3/delegations",
+  "/policy/v3/deliveries/authorize",
 ];
 const workflow = readFileSync(".github/workflows/docker.yml", "utf8");
 const source = readFileSync("tinycloud-node-server/src/policy_v3.rs", "utf8");
+const main = readFileSync("tinycloud-node-server/src/main.rs", "utf8");
 const probeStart = workflow.indexOf("- name: Verify deployed Policy/v3 routes");
 const probeEnd = workflow.indexOf("\n      - name:", probeStart + 1);
 
@@ -34,10 +36,76 @@ for (const invariant of [
   'NODE_ORIGIN="https://tee.node.tinycloud.xyz"',
   '"${NODE_ORIGIN}/version"',
   "--data '{}'",
-  '"${STATUS}" != "400"',
-  '"${STATUS}" != "422"',
+  'if [ "${route}" = "/policy/v3/deliveries/authorize" ]',
+  'EXPECTED="401"',
+  'EXPECTED="400 or 422"',
+  'case "${STATUS}" in 401)',
+  'case "${STATUS}" in 400|422)',
 ]) {
   if (!probe.includes(invariant)) {
     throw new Error(`Policy/v3 deployment probe lost required invariant: ${invariant}`);
   }
 }
+if (probe.includes("400|401|422")) {
+  throw new Error("unsigned delivery probe must require exactly HTTP 401");
+}
+
+const imagePinStart = workflow.indexOf("- name: Validate and pin deployment image");
+const preflightStart = workflow.indexOf("- name: Preflight sealed runtime configuration");
+const deployStart = workflow.indexOf("- name: Deploy to Phala Cloud");
+const preflightEnd = workflow.indexOf("\n      - name:", preflightStart + 1);
+if (
+  imagePinStart === -1 ||
+  preflightStart === -1 ||
+  deployStart === -1 ||
+  imagePinStart > preflightStart ||
+  preflightStart > deployStart
+) {
+  throw new Error("immutable image pin and runtime preflight must precede the Phala deploy");
+}
+const imagePinEnd = workflow.indexOf("\n      - name:", imagePinStart + 1);
+const imagePin = workflow.slice(imagePinStart, imagePinEnd === -1 ? undefined : imagePinEnd);
+for (const invariant of [
+  "outputs:\n      digest: ${{ steps.build.outputs.digest }}",
+  "id: build",
+  "BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }}",
+  'RESOLVED_IMAGE="${REGISTRY}/${IMAGE_NAME}@${DIGEST}"',
+  'docker pull "${RESOLVED_IMAGE}"',
+  '"${REVISION}" != "${GITHUB_SHA}"',
+  'os.environ["RESOLVED_IMAGE"]',
+]) {
+  if (!workflow.includes(invariant)) {
+    throw new Error(`immutable candidate deployment invariant is missing: ${invariant}`);
+  }
+}
+if (workflow.includes("- name: Update compose with release tag")) {
+  throw new Error("deployment must not select a mutable release tag");
+}
+if (!imagePin.includes("@${DIGEST}")) {
+  throw new Error("compose image replacement must use the immutable build digest");
+}
+const preflight = workflow.slice(preflightStart, preflightEnd === -1 ? undefined : preflightEnd);
+for (const invariant of [
+  "TINYCLOUD_STORAGE__DATABASE",
+  "TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64",
+  "TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full",
+  "TINYCLOUD_KEYS__TYPE: Dstack",
+  "--network none",
+  "--read-only",
+  "--cap-drop ALL",
+  "--security-opt no-new-privileges",
+  "-e TINYCLOUD_STORAGE__DATABASE",
+  "TINYCLOUD_SHARE_EMAIL__ENABLED=true",
+  "-e TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE",
+  "-e TINYCLOUD_KEYS__TYPE",
+  "--validate-config",
+]) {
+  if (!preflight.includes(invariant)) {
+    throw new Error(`runtime configuration preflight lost required invariant: ${invariant}`);
+  }
+}
+for (const invariant of ["resolve_runtime_config", 'arg == "--validate-config"']) {
+  if (!main.includes(invariant)) {
+    throw new Error(`runtime configuration preflight binary support is missing: ${invariant}`);
+  }
+}

tinycloud-node-server/src/config.rs

@@ -1448,6 +1448,25 @@ mod tests {
         assert!(config.validate().is_ok());
     }
 
+    /// The deployed pre-TC-500 bundle used this syntactically valid legacy
+    /// audience. It must fail at the same startup gate the release preflight
+    /// invokes rather than being discovered after the CVM has been replaced.
+    #[cfg(not(feature = "mounted-fixture"))]
+    #[tokio::test]
+    async fn legacy_email_origin_is_startup_fatal() {
+        let mut config = enabled_config();
+        let mut document = bundle_document(&config);
+        document["emailOrigin"] = serde_json::Value::String("https://email.tinycloud.xyz".into());
+        let file = NamedTempFile::new().expect("temporary trust bundle");
+        fs::write(file.path(), serde_json::to_vec(&document).unwrap()).expect("trust bundle write");
+        config.trust_bundle_path = Some(file.path().display().to_string());
+
+        assert_eq!(
+            config.resolve_trust_bundle(),
+            Err("share email trust bundle is inconsistent")
+        );
+    }
+
     /// The required field must be a canonical HTTPS origin with no path,
     /// query, fragment, port, or credentials.
     #[cfg(not(feature = "mounted-fixture"))]
@@ -1461,8 +1480,6 @@ mod tests {
             "https://operator:secret@email.tinycloud.xyz",
             "https://email.tinycloud.xyz:8443",
             "email.tinycloud.xyz",
-            // Correct shape, but an unreviewed production audience.
-            "https://email.tinycloud.xyz",
             "https://api.share.tinycloud.xyz",
             "",
             // Caught by the placeholder scan rather than the origin shape.
@@ -1789,4 +1806,21 @@ mod tests {
             "full legacy v1 validation must still require authority_material_path"
         );
     }
+
+    /// The release preflight deliberately uses the production PostgreSQL URL,
+    /// but must remain a no-I/O validation. A URL without `verify-full` is a
+    /// startup refusal even on the policy-v3 path that omits v1 authority
+    /// material.
+    #[cfg(not(feature = "mounted-fixture"))]
+    #[tokio::test]
+    async fn v2_preflight_rejects_postgres_without_verify_full() {
+        let mut config = enabled_config();
+        config.authority_material_path = None;
+        let _trust_bundle = install_bundle(&mut config);
+
+        assert_eq!(
+            config.validate_for_v2_database("postgresql://user:password@db.example/share"),
+            Err("share email PostgreSQL requires sslmode=verify-full")
+        );
+    }
 }

tinycloud-node-server/src/lib.rs

@@ -220,6 +220,23 @@ pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
     app_with_control(config, &tinycloud_config, None).await
 }
 
+/// Resolve the startup-only configuration gates without opening a key, network
+/// or database connection. Release automation uses this through the binary's
+/// `--validate-config` mode before it replaces the running CVM.
+pub fn resolve_runtime_config(tinycloud_config: &Config) -> Result<Config> {
+    let mut resolved = tinycloud_config.clone();
+    resolved.storage.resolve();
+    resolved.share_email = resolved
+        .share_email
+        .resolve_trust_bundle()
+        .map_err(|error| anyhow::anyhow!(error))?;
+    resolved
+        .share_email
+        .validate_for_v2_database(resolved.storage.database())
+        .map_err(|error| anyhow::anyhow!(error))?;
+    Ok(resolved)
+}
+
 /// The public Node surface keeps policy admission separate from the generic
 /// delegation and invocation data plane. Share-specific data routes are not
 /// mounted.
@@ -274,16 +291,7 @@ pub async fn app_with_control(
     tinycloud_config: &Config,
     control: Option<ControlPlaneHandle>,
 ) -> Result<Rocket<Build>> {
-    let mut tinycloud_config = tinycloud_config.clone();
-    tinycloud_config.storage.resolve();
-    tinycloud_config.share_email = tinycloud_config
-        .share_email
-        .resolve_trust_bundle()
-        .map_err(|error| anyhow::anyhow!(error))?;
-    tinycloud_config
-        .share_email
-        .validate_for_v2_database(tinycloud_config.storage.database())
-        .map_err(|error| anyhow::anyhow!(error))?;
+    let tinycloud_config = resolve_runtime_config(tinycloud_config)?;
 
     // Ensure local storage directories exist.
     // SQLite file paths and local dirs are resources the server owns — auto-create them.

tinycloud-node-server/src/main.rs

@@ -10,7 +10,7 @@ use rocket::{
     figment::providers::{Env, Format, Serialized, Toml},
     tokio,
 };
-use tinycloud::{app, config, prometheus};
+use tinycloud::{app, config, prometheus, resolve_runtime_config};
 
 fn build_config_figment() -> rocket::figment::Figment {
     let config_file =
@@ -31,6 +31,28 @@ async fn main() {
     let config = build_config_figment(); // That's just for easy access to ROCKET_LOG_LEVEL
     let tinycloud_config = config.extract::<config::Config>().unwrap();
 
+    // This is deliberately before `app`: release automation needs to prove
+    // the exact candidate accepts the sealed trust bundle without opening the
+    // production database, resolving the node key, or binding a socket.
+    if std::env::args_os()
+        .skip(1)
+        .any(|arg| arg == "--validate-config")
+    {
+        match resolve_runtime_config(&tinycloud_config) {
+            Ok(_) => {
+                eprintln!("tinycloud-node runtime configuration is valid");
+                return;
+            }
+            Err(error) => {
+                eprintln!("tinycloud-node runtime configuration is invalid");
+                for cause in error.chain() {
+                    eprintln!("  {cause}");
+                }
+                std::process::exit(1);
+            }
+        }
+    }
+
     let rocket = match app(&config).await {
         Ok(r) => r.ignite().await.unwrap(),
         Err(e) => {