From b7c0034ba54ed57b94637e20b517e59da0594232 Mon Sep 17 00:00:00 2001 From: Samuel Gbafa Date: Tue, 15 Sep 2026 03:32:02 +0000 Subject: [PATCH 1/2] fix(tc-500): preflight sealed node configuration --- .github/workflows/docker.yml | 47 +++++++++++++++++++++- scripts/check-deployment-policy-probes.mjs | 35 +++++++++++++++- tinycloud-node-server/src/config.rs | 21 +++++++++- tinycloud-node-server/src/lib.rs | 28 ++++++++----- tinycloud-node-server/src/main.rs | 24 ++++++++++- 5 files changed, 138 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f62eac69..eb3316ee 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -232,9 +232,19 @@ jobs: runs-on: ubuntu-latest needs: [build, build-dstack] if: github.event_name == 'release' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.deploy_phala) + permissions: + contents: read + packages: read steps: - uses: actions/checkout@v4 + - name: Log in to GHCR for runtime configuration preflight + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Node.js uses: actions/setup-node@v4 with: @@ -277,6 +287,26 @@ jobs: echo "Resolved image tag: ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}" cat docker-compose.dstack-postgres.yaml + # TC-500: Phala considers a CVM ready before compose services have + # started. Validate the production-sealed bundle using the exact image + # selected above, without database, dstack, network, or socket access, + # before the deploy step can replace the healthy CVM. + - name: Preflight sealed runtime configuration + env: + TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: ${{ secrets.PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64 }} + run: | + set -euo pipefail + IMAGE="$(sed -n -E 's/^ image: (ghcr.io\/tinycloudlabs\/tinycloud-node(:[^[:space:]]+|@sha256:[0-9a-f]{64}))$/\1/p' docker-compose.dstack-postgres.yaml)" + if [ "$(printf '%s\n' "${IMAGE}" | awk 'NF { count += 1 } END { print count + 0 }')" != "1" ]; then + echo "::error::Expected exactly one resolved TinyCloud Node image for runtime preflight" + exit 1 + fi + docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges \ + -e TINYCLOUD_SHARE_EMAIL__ENABLED=true \ + -e TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64 \ + "${IMAGE}" --validate-config + - name: Deploy to Phala Cloud env: PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} @@ -408,6 +438,7 @@ jobs: /policy/v3/policies /policy/v3/challenges /policy/v3/delegations + /policy/v3/deliveries/authorize ) # The direct TEE ingress obtains/loads its certificate after the @@ -435,8 +466,20 @@ jobs: --data '{}' \ "${NODE_ORIGIN}${route}")" - if [ "${STATUS}" != "400" ] && [ "${STATUS}" != "422" ]; then - echo "::error::${route} returned HTTP ${STATUS}; expected 400 or 422 for an intentionally invalid request. The Policy/v3 production contract is not mounted correctly." + # Delivery is authenticated before request-body validation, so an + # intentionally unsigned body is expected to return 401. Other + # control-plane routes reject the body first. Neither case may + # silently accept a missing route or a server failure. + if [ "${route}" = "/policy/v3/deliveries/authorize" ]; then + EXPECTED="400, 401, or 422" + case "${STATUS}" in 400|401|422) VALID=true ;; *) VALID=false ;; esac + else + EXPECTED="400 or 422" + case "${STATUS}" in 400|422) VALID=true ;; *) VALID=false ;; esac + fi + + if [ "${VALID}" != true ]; then + echo "::error::${route} returned HTTP ${STATUS}; expected ${EXPECTED} for an intentionally invalid request. The Policy/v3 production contract is not mounted correctly." head -c 1000 "${RESPONSE_FILE}" || true rm -f "${RESPONSE_FILE}" exit 1 diff --git a/scripts/check-deployment-policy-probes.mjs b/scripts/check-deployment-policy-probes.mjs index 7c9072d5..d88c1701 100644 --- a/scripts/check-deployment-policy-probes.mjs +++ b/scripts/check-deployment-policy-probes.mjs @@ -5,9 +5,11 @@ const expectedRoutes = [ "/policy/v3/policies", "/policy/v3/challenges", "/policy/v3/delegations", + "/policy/v3/deliveries/authorize", ]; const workflow = readFileSync(".github/workflows/docker.yml", "utf8"); const source = readFileSync("tinycloud-node-server/src/policy_v3.rs", "utf8"); +const main = readFileSync("tinycloud-node-server/src/main.rs", "utf8"); const probeStart = workflow.indexOf("- name: Verify deployed Policy/v3 routes"); const probeEnd = workflow.indexOf("\n - name:", probeStart + 1); @@ -34,10 +36,39 @@ for (const invariant of [ 'NODE_ORIGIN="https://tee.node.tinycloud.xyz"', '"${NODE_ORIGIN}/version"', "--data '{}'", - '"${STATUS}" != "400"', - '"${STATUS}" != "422"', + 'if [ "${route}" = "/policy/v3/deliveries/authorize" ]', + 'EXPECTED="400, 401, or 422"', + 'EXPECTED="400 or 422"', + 'case "${STATUS}" in 400|401|422)', + 'case "${STATUS}" in 400|422)', ]) { if (!probe.includes(invariant)) { throw new Error(`Policy/v3 deployment probe lost required invariant: ${invariant}`); } } + +const preflightStart = workflow.indexOf("- name: Preflight sealed runtime configuration"); +const deployStart = workflow.indexOf("- name: Deploy to Phala Cloud"); +const preflightEnd = workflow.indexOf("\n - name:", preflightStart + 1); +if (preflightStart === -1 || deployStart === -1 || preflightStart > deployStart) { + throw new Error("sealed runtime configuration preflight must precede the Phala deploy"); +} +const preflight = workflow.slice(preflightStart, preflightEnd === -1 ? undefined : preflightEnd); +for (const invariant of [ + "TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64", + "--network none", + "--read-only", + "--cap-drop ALL", + "--security-opt no-new-privileges", + "TINYCLOUD_SHARE_EMAIL__ENABLED=true", + "--validate-config", +]) { + if (!preflight.includes(invariant)) { + throw new Error(`runtime configuration preflight lost required invariant: ${invariant}`); + } +} +for (const invariant of ["resolve_runtime_config", 'arg == "--validate-config"']) { + if (!main.includes(invariant)) { + throw new Error(`runtime configuration preflight binary support is missing: ${invariant}`); + } +} diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 1ab7ce97..ca26b516 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1448,6 +1448,25 @@ mod tests { assert!(config.validate().is_ok()); } + /// The deployed pre-TC-500 bundle used this syntactically valid legacy + /// audience. It must fail at the same startup gate the release preflight + /// invokes rather than being discovered after the CVM has been replaced. + #[cfg(not(feature = "mounted-fixture"))] + #[tokio::test] + async fn legacy_email_origin_is_startup_fatal() { + let mut config = enabled_config(); + let mut document = bundle_document(&config); + document["emailOrigin"] = serde_json::Value::String("https://email.tinycloud.xyz".into()); + let file = NamedTempFile::new().expect("temporary trust bundle"); + fs::write(file.path(), serde_json::to_vec(&document).unwrap()).expect("trust bundle write"); + config.trust_bundle_path = Some(file.path().display().to_string()); + + assert_eq!( + config.resolve_trust_bundle(), + Err("share email trust bundle is inconsistent") + ); + } + /// The required field must be a canonical HTTPS origin with no path, /// query, fragment, port, or credentials. #[cfg(not(feature = "mounted-fixture"))] @@ -1461,8 +1480,6 @@ mod tests { "https://operator:secret@email.tinycloud.xyz", "https://email.tinycloud.xyz:8443", "email.tinycloud.xyz", - // Correct shape, but an unreviewed production audience. - "https://email.tinycloud.xyz", "https://api.share.tinycloud.xyz", "", // Caught by the placeholder scan rather than the origin shape. diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index b873c755..44ee796c 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -220,6 +220,23 @@ pub async fn app(config: &Figment) -> Result> { app_with_control(config, &tinycloud_config, None).await } +/// Resolve the startup-only configuration gates without opening a key, network +/// or database connection. Release automation uses this through the binary's +/// `--validate-config` mode before it replaces the running CVM. +pub fn resolve_runtime_config(tinycloud_config: &Config) -> Result { + let mut resolved = tinycloud_config.clone(); + resolved.storage.resolve(); + resolved.share_email = resolved + .share_email + .resolve_trust_bundle() + .map_err(|error| anyhow::anyhow!(error))?; + resolved + .share_email + .validate_for_v2_database(resolved.storage.database()) + .map_err(|error| anyhow::anyhow!(error))?; + Ok(resolved) +} + /// The public Node surface keeps policy admission separate from the generic /// delegation and invocation data plane. Share-specific data routes are not /// mounted. @@ -274,16 +291,7 @@ pub async fn app_with_control( tinycloud_config: &Config, control: Option, ) -> Result> { - let mut tinycloud_config = tinycloud_config.clone(); - tinycloud_config.storage.resolve(); - tinycloud_config.share_email = tinycloud_config - .share_email - .resolve_trust_bundle() - .map_err(|error| anyhow::anyhow!(error))?; - tinycloud_config - .share_email - .validate_for_v2_database(tinycloud_config.storage.database()) - .map_err(|error| anyhow::anyhow!(error))?; + let tinycloud_config = resolve_runtime_config(tinycloud_config)?; // Ensure local storage directories exist. // SQLite file paths and local dirs are resources the server owns — auto-create them. diff --git a/tinycloud-node-server/src/main.rs b/tinycloud-node-server/src/main.rs index faa63d6e..5a112f1a 100644 --- a/tinycloud-node-server/src/main.rs +++ b/tinycloud-node-server/src/main.rs @@ -10,7 +10,7 @@ use rocket::{ figment::providers::{Env, Format, Serialized, Toml}, tokio, }; -use tinycloud::{app, config, prometheus}; +use tinycloud::{app, config, prometheus, resolve_runtime_config}; fn build_config_figment() -> rocket::figment::Figment { let config_file = @@ -31,6 +31,28 @@ async fn main() { let config = build_config_figment(); // That's just for easy access to ROCKET_LOG_LEVEL let tinycloud_config = config.extract::().unwrap(); + // This is deliberately before `app`: release automation needs to prove + // the exact candidate accepts the sealed trust bundle without opening the + // production database, resolving the node key, or binding a socket. + if std::env::args_os() + .skip(1) + .any(|arg| arg == "--validate-config") + { + match resolve_runtime_config(&tinycloud_config) { + Ok(_) => { + eprintln!("tinycloud-node runtime configuration is valid"); + return; + } + Err(error) => { + eprintln!("tinycloud-node runtime configuration is invalid"); + for cause in error.chain() { + eprintln!(" {cause}"); + } + std::process::exit(1); + } + } + } + let rocket = match app(&config).await { Ok(r) => r.ignite().await.unwrap(), Err(e) => { From 55761a1025a7f5ace2a9e28aaea2a4f6be05a2a5 Mon Sep 17 00:00:00 2001 From: Samuel Gbafa Date: Tue, 15 Sep 2026 03:47:28 +0000 Subject: [PATCH 2/2] fix(tc-500): pin Phala candidate by digest --- .github/workflows/docker.yml | 73 +++++++++++++--------- scripts/check-deployment-policy-probes.mjs | 45 +++++++++++-- tinycloud-node-server/src/config.rs | 17 +++++ 3 files changed, 102 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eb3316ee..a90beb41 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -167,6 +167,8 @@ jobs: runs-on: ubuntu-latest needs: [version-guard] if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'push' && github.ref == 'refs/heads/main') + outputs: + digest: ${{ steps.build.outputs.digest }} permissions: contents: read packages: write @@ -218,6 +220,7 @@ jobs: type=raw,value=dstack${{ steps.build_features.outputs.image_suffix }},enable={{is_default_branch}} - name: Build and push dstack Docker image + id: build uses: docker/build-push-action@v5 with: context: . @@ -230,7 +233,7 @@ jobs: deploy-phala: runs-on: ubuntu-latest - needs: [build, build-dstack] + needs: [version-guard, build, build-dstack] if: github.event_name == 'release' || startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.deploy_phala) permissions: contents: read @@ -258,42 +261,51 @@ jobs: - name: Install Phala CLI run: npm install -g phala@1.1.19 - - name: Update compose with release tag + - name: Validate and pin deployment image + env: + BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }} run: | - if [ "${{ github.event_name }}" = "release" ]; then - TAG="${{ github.event.release.tag_name }}" - else - TAG="${{ inputs.image_version }}" - if [ -z "${TAG}" ]; then - if [ "${GITHUB_REF_TYPE}" != "tag" ]; then - echo "::error::workflow_dispatch deploy_phala requires image_version unless the workflow is run from a tag" - exit 1 - fi - TAG="${GITHUB_REF_NAME}" - fi + set -euo pipefail + DIGEST="${BUILD_DIGEST}" + if ! [[ "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::build-dstack did not produce an immutable OCI digest" + exit 1 fi - # metadata-action emits {{version}} without the leading v (e.g. 1.3.0). - # Strip a leading v from the tag to match the pushed image tag. - VERSION="${TAG#v}" - DUCKDB_SUFFIX="" - if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.include_duckdb }}" = "true" ]; then - DUCKDB_SUFFIX="-duckdb" + export RESOLVED_IMAGE="${REGISTRY}/${IMAGE_NAME}@${DIGEST}" + docker pull "${RESOLVED_IMAGE}" + REVISION="$(docker image inspect "${RESOLVED_IMAGE}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" + if [ "${REVISION}" != "${GITHUB_SHA}" ]; then + echo "::error::candidate OCI revision '${REVISION}' does not match workflow commit '${GITHUB_SHA}'" + exit 1 fi - # The prod CVM uses the dstack-suffixed image. Replace the floating - # ":dstack" tag in the checked-in compose with the versioned tag - # built by the build-dstack job (e.g. ":1.3.0-dstack" or - # ":1.3.0-dstack-duckdb"). - sed -i "s|ghcr.io/tinycloudlabs/tinycloud-node:dstack|ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}|g" docker-compose.dstack-postgres.yaml - echo "Resolved image tag: ghcr.io/tinycloudlabs/tinycloud-node:${VERSION}-dstack${DUCKDB_SUFFIX}" + + # Keep this immutable reference through preflight, compose parsing, + # and Phala. A version tag can be retargeted between build and deploy. + python3 - <<'PY' + from pathlib import Path + import os + + path = Path("docker-compose.dstack-postgres.yaml") + content = path.read_text() + floating = "ghcr.io/tinycloudlabs/tinycloud-node:dstack" + if content.count(floating) != 1: + raise SystemExit("expected exactly one floating dstack image in compose") + path.write_text(content.replace(floating, os.environ["RESOLVED_IMAGE"])) + PY + echo "Resolved immutable candidate: ${RESOLVED_IMAGE}" cat docker-compose.dstack-postgres.yaml # TC-500: Phala considers a CVM ready before compose services have # started. Validate the production-sealed bundle using the exact image - # selected above, without database, dstack, network, or socket access, - # before the deploy step can replace the healthy CVM. + # selected above against the production database/TLS/key configuration, + # without opening a database, resolving dstack, network, or socket + # access, before the deploy step can replace the healthy CVM. - name: Preflight sealed runtime configuration env: + TINYCLOUD_STORAGE__DATABASE: ${{ secrets.PROD_TINYCLOUD_DATABASE_URL }} TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: ${{ secrets.PROD_TINYCLOUD_SHARE_TRUST_BUNDLE_BASE64 }} + TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full + TINYCLOUD_KEYS__TYPE: Dstack run: | set -euo pipefail IMAGE="$(sed -n -E 's/^ image: (ghcr.io\/tinycloudlabs\/tinycloud-node(:[^[:space:]]+|@sha256:[0-9a-f]{64}))$/\1/p' docker-compose.dstack-postgres.yaml)" @@ -303,8 +315,11 @@ jobs: fi docker run --rm --network none --read-only --cap-drop ALL \ --security-opt no-new-privileges \ + -e TINYCLOUD_STORAGE__DATABASE \ -e TINYCLOUD_SHARE_EMAIL__ENABLED=true \ -e TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64 \ + -e TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE \ + -e TINYCLOUD_KEYS__TYPE \ "${IMAGE}" --validate-config - name: Deploy to Phala Cloud @@ -471,8 +486,8 @@ jobs: # control-plane routes reject the body first. Neither case may # silently accept a missing route or a server failure. if [ "${route}" = "/policy/v3/deliveries/authorize" ]; then - EXPECTED="400, 401, or 422" - case "${STATUS}" in 400|401|422) VALID=true ;; *) VALID=false ;; esac + EXPECTED="401" + case "${STATUS}" in 401) VALID=true ;; *) VALID=false ;; esac else EXPECTED="400 or 422" case "${STATUS}" in 400|422) VALID=true ;; *) VALID=false ;; esac diff --git a/scripts/check-deployment-policy-probes.mjs b/scripts/check-deployment-policy-probes.mjs index d88c1701..f14a9b55 100644 --- a/scripts/check-deployment-policy-probes.mjs +++ b/scripts/check-deployment-policy-probes.mjs @@ -37,30 +37,67 @@ for (const invariant of [ '"${NODE_ORIGIN}/version"', "--data '{}'", 'if [ "${route}" = "/policy/v3/deliveries/authorize" ]', - 'EXPECTED="400, 401, or 422"', + 'EXPECTED="401"', 'EXPECTED="400 or 422"', - 'case "${STATUS}" in 400|401|422)', + 'case "${STATUS}" in 401)', 'case "${STATUS}" in 400|422)', ]) { if (!probe.includes(invariant)) { throw new Error(`Policy/v3 deployment probe lost required invariant: ${invariant}`); } } +if (probe.includes("400|401|422")) { + throw new Error("unsigned delivery probe must require exactly HTTP 401"); +} +const imagePinStart = workflow.indexOf("- name: Validate and pin deployment image"); const preflightStart = workflow.indexOf("- name: Preflight sealed runtime configuration"); const deployStart = workflow.indexOf("- name: Deploy to Phala Cloud"); const preflightEnd = workflow.indexOf("\n - name:", preflightStart + 1); -if (preflightStart === -1 || deployStart === -1 || preflightStart > deployStart) { - throw new Error("sealed runtime configuration preflight must precede the Phala deploy"); +if ( + imagePinStart === -1 || + preflightStart === -1 || + deployStart === -1 || + imagePinStart > preflightStart || + preflightStart > deployStart +) { + throw new Error("immutable image pin and runtime preflight must precede the Phala deploy"); +} +const imagePinEnd = workflow.indexOf("\n - name:", imagePinStart + 1); +const imagePin = workflow.slice(imagePinStart, imagePinEnd === -1 ? undefined : imagePinEnd); +for (const invariant of [ + "outputs:\n digest: ${{ steps.build.outputs.digest }}", + "id: build", + "BUILD_DIGEST: ${{ needs.build-dstack.outputs.digest }}", + 'RESOLVED_IMAGE="${REGISTRY}/${IMAGE_NAME}@${DIGEST}"', + 'docker pull "${RESOLVED_IMAGE}"', + '"${REVISION}" != "${GITHUB_SHA}"', + 'os.environ["RESOLVED_IMAGE"]', +]) { + if (!workflow.includes(invariant)) { + throw new Error(`immutable candidate deployment invariant is missing: ${invariant}`); + } +} +if (workflow.includes("- name: Update compose with release tag")) { + throw new Error("deployment must not select a mutable release tag"); +} +if (!imagePin.includes("@${DIGEST}")) { + throw new Error("compose image replacement must use the immutable build digest"); } const preflight = workflow.slice(preflightStart, preflightEnd === -1 ? undefined : preflightEnd); for (const invariant of [ + "TINYCLOUD_STORAGE__DATABASE", "TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64", + "TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full", + "TINYCLOUD_KEYS__TYPE: Dstack", "--network none", "--read-only", "--cap-drop ALL", "--security-opt no-new-privileges", + "-e TINYCLOUD_STORAGE__DATABASE", "TINYCLOUD_SHARE_EMAIL__ENABLED=true", + "-e TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE", + "-e TINYCLOUD_KEYS__TYPE", "--validate-config", ]) { if (!preflight.includes(invariant)) { diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index ca26b516..fc1584dc 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1806,4 +1806,21 @@ mod tests { "full legacy v1 validation must still require authority_material_path" ); } + + /// The release preflight deliberately uses the production PostgreSQL URL, + /// but must remain a no-I/O validation. A URL without `verify-full` is a + /// startup refusal even on the policy-v3 path that omits v1 authority + /// material. + #[cfg(not(feature = "mounted-fixture"))] + #[tokio::test] + async fn v2_preflight_rejects_postgres_without_verify_full() { + let mut config = enabled_config(); + config.authority_material_path = None; + let _trust_bundle = install_bundle(&mut config); + + assert_eq!( + config.validate_for_v2_database("postgresql://user:password@db.example/share"), + Err("share email PostgreSQL requires sslmode=verify-full") + ); + } }