diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index ee688314bd..2d62e43afe 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -104,6 +104,8 @@ The Windows build lane is implemented by these tracked files: | `tasks/rust.toml`, `tasks/test.toml`, and `tasks/markdown.toml` | Windows routing for compiler-bearing checks, explicit Unix-only test skips, and Markdown dependency setup. | | `tasks/scripts/windows-msvc.ps1` | PowerShell wrapper that enters the Visual Studio developer environment and invokes Cargo. | | `.github/workflows/windows-msvc.yml` | Opt-in PR lint and test plus advisory `windows` branch cache seeding and dependent binary builds on native x64 and ARM64 runners. | +| `tasks/windows-z3.json` | Pinned Z3 source, package digests, and package compiler baseline. | +| `tasks/scripts/windows-z3.ps1` | Native HTTPS download, integrity validation, and static dependency caching. | | `.agents/skills/build-openshell-mxc-windows/` | This skill and companion reference material. | Use the code that is already in the repo. Do not generate a parallel Windows @@ -203,9 +205,12 @@ from this skill. | `OPENSHELL_MXC_SKIP_ARM64` | `0` | Set to `1` to skip ARM64 when using `all` tasks. | | `OPENSHELL_WINDOWS_BUILD_JOBS` | `CARGO_BUILD_JOBS`, then `4` | Positive Cargo job limit used by the wrapper. | | `CARGO_TARGET_DIR` | `target` under repo root | Override Cargo output location. Use a short absolute path when x64-to-ARM64 builds approach Windows path-length limits. | -| `Z3_LIBRARY_PATH_OVERRIDE` | unset | Directory containing target-compatible system `libz3.lib` and `libz3.dll`, used instead of the prebuilt download. | +| `OPENSHELL_WINDOWS_STATIC_RUNTIME` | `1` | Static CRT/Z3 by default; reuse the pinned GHCR `/MT` package. Set `0` only for legacy shared builds. | +| `OPENSHELL_WINDOWS_Z3_SOURCE` | `0` | Explicitly build pinned Z3 source, overriding a library override. | +| `Z3_SRC_SOURCE_DIR` | unset | Local Z3 5.1.0 source tree; selects source when no library override is supplied. | +| `Z3_LIBRARY_PATH_OVERRIDE` | unset | Directory containing a target-compatible static `/MT` `libz3.lib`; legacy shared overrides apply only in explicit shared mode. | | `Z3_SYS_Z3_HEADER` | unset | Full `z3.h` path required with a system Z3 library. | -| `Z3_SYS_Z3_VERSION` | `5.1.0` | Pinned official prebuilt Z3 release selected by the wrapper. | +| `Z3_SYS_Z3_VERSION` | `5.1.0` | Solver version pinned with source and package digests in `tasks/windows-z3.json`. | | `READ_ONLY_GITHUB_TOKEN` | unset | Optional token for the Z3 release lookup; GitHub Actions supplies `github.token`. | | `RUSTC_WRAPPER` | inherited | The wrapper resolves an available command to an absolute path. If it is unavailable, the wrapper warns and continues without compiler caching. | @@ -304,7 +309,7 @@ jobs in the current mirror push run, or push a new mirrored commit. The binaries The ARM64 check/build steps in this x64-host contract are cross-builds. The wrapper discovers and adds host-native LLVM and Ninja to `PATH`, requires the ARM64 compiler and Spectre-mitigated libraries, lets ARM64 crypto crates select -`clang-cl`, and downloads the official prebuilt ARM64 Z3 package. +`clang-cl`, and downloads the pinned static ARM64 Z3 package from GHCR. On ARM64 hosts, validate the native ARM64 check, build, and test path. The wrapper rejects test targets that do not match the host architecture, so x64 @@ -423,14 +428,11 @@ architecture through Windows system configuration (not an emulated SSH shell), caches uploads by SHA256, and runs the unchanged real E2E harness. Results and host probes return to `target/windows-remote-results`, including nonzero test outcomes. The remote host needs SSH, OpenSSL, and MXC, not a compiler toolchain. -Package the matching `libz3.dll` from the target-specific Cargo cache alongside -the gateway: Windows prebuilt Z3 uses an import library, not static linkage. -The `windows:build:*` tasks now stage the pinned DLL, check PE architecture, -reject conflicting cache hashes, and verify the copied hash. Local MXC E2E -requires the adjacent DLL and reports a preflight error if it is missing. -Use `-Z3DllPath` when discovery is ambiguous. The native Visual C++ runtime -must be present on the remote host. Verify DLL architecture and the gateway -loader before E2E; do not count missing-DLL crashes as MXC enforcement failures. +Default builds embed Z3 and the CRT. The remote helper reads gateway imports +and includes the matching `libz3.dll` only for shared builds. `-Z3DllPath` +continues to select a shared DLL when discovery is ambiguous; those builds +also need the native Visual C++ runtime. Check gateway startup before E2E; +do not count loader failures as MXC enforcement failures. Verify the SSH host key beforehand. Never send the private SSH key or bundle the harness signing-key directories. `-SkipBuild` is an explicit stale-artifact opt-in, not evidence that artifacts match current sources. See the MXC driver @@ -591,3 +593,32 @@ ETW qualification. Build native release binaries first and set membership and leaves a results bundle under the target directory. Ordinary MXC E2E success does not establish ETW audit coverage. Coverage must use structured JSONL and fail when an expected event is absent or has the wrong OCSF class. + +## Static Windows ZIP packages + +Windows tasks and CI embed the CRT and Z3 by default. The wrapper loads +`.cargo/windows-static.toml` and downloads the matching static `/MT` package +from `ghcr.io/nvidia/openshell/z3`. `tasks/windows-z3.json` pins Z3 5.1.0, +the source checksum, and each target's OCI, archive, and build-manifest digests. +Native PowerShell performs HTTPS downloads without Docker, ORAS, or GitHub +credentials. `.cache/windows-z3` is cached separately from Rust outputs. + +Source compilation occurs only when the package is unavailable or explicitly +requested. Digest/version/architecture mismatches fail. Preserve Ninja, +native Release optimization, source caching outside Cargo pruning, and scoped +stale-generator invalidation. Supplied static `/MT` libraries remain supported. +`OPENSHELL_WINDOWS_STATIC_RUNTIME=0` selects legacy shared behavior; an inherited +shared override otherwise produces a warning and selects the pinned package. + +Static release builds audit all four executables' normal/delayed imports and +remove stale staged Z3 DLLs. The remote helper packages the DLL only for shared +builds and preserves `-Z3DllPath` compatibility. The `static_runtime` manual +workflow input defaults to true. Ordinary PR CI keeps its lint/test workload; +it does not build Z3 when the pinned package or verified cache is available. + +The explicit `publish_z3` Windows MSVC workflow input builds only the two +native dependency packages and publishes versioned OCI tags. The producer explicitly selects MSVC 14.44 and uses +Release `/MT` without LTO, audits solver imports, and checks SAT/UNSAT before +packaging libraries, headers, license, and build metadata. The workflow refuses to +replace existing tags with different content. Update pins only after verifying public package downloads. +See [the contributor build options](../../../CONTRIBUTING.md#windows-full-build). diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index f157d87951..f0fa6925da 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -93,9 +93,9 @@ For ARM64, verify the Visual Studio instance contains the ARM64 MSVC tools, ARM64 Spectre-mitigated libraries, Clang tools, CMake tools, and a Windows SDK. Clang supplies host-native `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies such as `aws-lc-sys`. Native and -x64-to-ARM64 builds use the official prebuilt Z3 5.1.0 import library and -`libz3.dll` for the target architecture. Release build tasks stage that DLL -beside the four executables; copy it with them to the runtime host. +x64-to-ARM64 builds use the pinned Z3 5.1.0 static `/MT` GHCR package for the +target architecture. Default builds embed Z3 and the CRT. Explicit shared mode +stages the matching DLL beside the executables. An x64-to-ARM64 check/build discovers and adds host-native Ninja to `PATH`, while the crypto crates select `clang-cl`. Use a short `CARGO_TARGET_DIR` if Windows path-length limits are reached. @@ -269,3 +269,9 @@ if ($arch -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Run the full x64-host `windows:ci` lane when build or test behavior changed and the host can run that lane natively. + +Static CRT/Z3 is the default. Reuse the pinned GHCR `/MT` package or supply a +static library/header override. Build checksum-pinned 5.1.0 source only when +requested or the package is unavailable. `OPENSHELL_WINDOWS_STATIC_RUNTIME=0` +retains explicit legacy shared behavior. See `CONTRIBUTING.md` for package +publication and pin updates. diff --git a/.cargo/windows-msvc.toml b/.cargo/windows-msvc.toml new file mode 100644 index 0000000000..7b1c05f2fe --- /dev/null +++ b/.cargo/windows-msvc.toml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Loaded only by the Windows MSVC wrapper. Keep native Z3 optimized, as the +# previous prebuilt DLL was, without changing Rust application debug profiles. +# z3-sys forwards its OPT_LEVEL/DEBUG settings to the native CMake build. +[profile.dev.package.z3-sys] +opt-level = 3 +debug = false + +[profile.test.package.z3-sys] +opt-level = 3 +debug = false diff --git a/.cargo/windows-static.toml b/.cargo/windows-static.toml new file mode 100644 index 0000000000..813e7c0b48 --- /dev/null +++ b/.cargo/windows-static.toml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Explicit Windows package builds only. cc/cmake also consume crt-static. +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] + +[target.aarch64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] diff --git a/.github/workflows/windows-msvc.yml b/.github/workflows/windows-msvc.yml index a268629a22..94b98f61e6 100644 --- a/.github/workflows/windows-msvc.yml +++ b/.github/workflows/windows-msvc.yml @@ -6,18 +6,28 @@ on: - windows - "pull-request/[0-9]+" workflow_dispatch: + inputs: + static_runtime: + description: Embed static CRT/Z3 (disable only for legacy shared builds) + type: boolean + default: true + publish_z3: + description: Build and publish pinned static Z3 dependency packages only + type: boolean + default: false permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.publish_z3 && 'z3' || 'checks' }} cancel-in-progress: true env: # sccache cannot cache incremental Rust compilations, and rust-cache removes # incremental artifacts before saving the target directory. CARGO_INCREMENTAL: "0" + OPENSHELL_WINDOWS_STATIC_RUNTIME: ${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '1' || '0' }} jobs: pr_metadata: @@ -27,12 +37,12 @@ jobs: contents: read pull-requests: read outputs: - should_run: ${{ github.ref == 'refs/heads/windows' || steps.gate.outputs.should_run == 'true' }} + should_run: ${{ !inputs.publish_z3 && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/windows' || steps.gate.outputs.should_run == 'true') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - id: gate - if: github.ref != 'refs/heads/windows' + if: github.ref != 'refs/heads/windows' && github.event_name != 'workflow_dispatch' uses: ./.github/actions/pr-gate with: required_label: test:windows @@ -58,11 +68,16 @@ jobs: env: READ_ONLY_GITHUB_TOKEN: ${{ github.token }} SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: windows-msvc-${{ matrix.arch }} + SCCACHE_GHA_VERSION: windows-msvc${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '-static-v1' || '' }}-${{ matrix.arch }} SCCACHE_CLIENT_SIDE: "1" SCCACHE_BASEDIRS: ${{ github.workspace }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Cache pinned Z3 package + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .cache/windows-z3 + key: windows-z3-${{ matrix.rust_target }}-${{ hashFiles('tasks/windows-z3.json') }} - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 with: install: false @@ -90,10 +105,16 @@ jobs: # wrapper; build steps still inherit sccache from the job. RUSTC_WRAPPER: "" with: - shared-key: windows-msvc-${{ matrix.arch }} + shared-key: windows-msvc${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '-static-v3' || '' }}-${{ matrix.arch }} cache-targets: "true" cache-on-failure: "true" cache-bin: "false" + - name: Test Windows dependency helpers + run: | + powershell -NoProfile -File tasks/scripts/windows-runtime-tests.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + powershell -NoProfile -File tasks/scripts/windows-z3-tests.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Lint Windows-supported crates run: mise run --skip-tools windows:lint:${{ matrix.arch }} - name: Test @@ -102,6 +123,110 @@ jobs: if: always() run: sccache --show-stats + z3-package: + name: Package pinned Z3 (${{ matrix.arch }}) + if: github.event_name == 'workflow_dispatch' && inputs.publish_z3 + strategy: + fail-fast: false + matrix: + include: + - arch: x64 + runner: windows-2025 + rust_target: x86_64-pc-windows-msvc + - arch: arm64 + runner: windows-11-arm + rust_target: aarch64-pc-windows-msvc + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Build and validate static Z3 package + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/z3 + run: powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 package-z3 ${{ matrix.rust_target }} + - name: Upload verified package + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: windows-z3-${{ matrix.arch }} + path: ${{ runner.temp }}/z3/windows-z3-packages/* + compression-level: 0 + - name: Upload dependency build log + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: windows-z3-build-log-${{ matrix.arch }} + path: package-z3-${{ matrix.rust_target }}.log + + publish-z3: + name: Publish pinned Z3 to GHCR + needs: z3-package + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: windows-z3-* + merge-multiple: true + - uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.0 + - name: Log in to GHCR + env: + GHCR_TOKEN: ${{ github.token }} + run: printf '%s' "$GHCR_TOKEN" | oras login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + - name: Publish versioned packages + env: + SOURCE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + version="$(jq -r .version tasks/windows-z3.json)" + revision="$(jq -r .packageRevision tasks/windows-z3.json)" + repository="$(jq -r .repository tasks/windows-z3.json)" + for target in x86_64-pc-windows-msvc aarch64-pc-windows-msvc; do + archive="z3-${version}-${target}-mt-v${revision}.zip" + sha256sum --check "${archive}.sha256" + unzip -p "$archive" build.json | sha256sum > "${archive}.build-manifest.sha256" + ref="${repository}:${version}-${target}-mt-v${revision}" + if existing="$(oras resolve "$ref" 2>resolve-error.txt)"; then + read -r archive_sha _ < "${archive}.sha256" + oras manifest fetch "${repository}@${existing}" > existing-manifest.json + if ! jq -e --arg hash "${archive_sha,,}" \ + '.artifactType == "application/vnd.nvidia.openshell.z3.v1" and (.layers | length) == 1 and .layers[0].digest == ("sha256:" + $hash)' \ + existing-manifest.json; then + echo "Refusing to replace existing dependency tag: $ref" >&2 + exit 1 + fi + echo "Reusing the identical published package: $ref" + else + if ! grep -Eqi 'NAME_UNKNOWN|MANIFEST_UNKNOWN|not found' resolve-error.txt; then + cat resolve-error.txt >&2 + exit 1 + fi + oras push "$ref" \ + --artifact-type application/vnd.nvidia.openshell.z3.v1 \ + --annotation org.opencontainers.image.source=https://github.com/NVIDIA/OpenShell \ + --annotation "org.opencontainers.image.revision=${SOURCE_SHA}" \ + --annotation "org.opencontainers.image.version=${version}" \ + --annotation org.opencontainers.image.licenses=MIT \ + "${archive}:application/zip" + fi + digest="$(oras resolve "$ref")" + printf '%s@%s\n' "$ref" "$digest" | tee -a published-z3.txt + printf '%s %s\n' "$digest" "$ref" >> "$GITHUB_STEP_SUMMARY" + done + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: published-z3-pins + path: | + published-z3.txt + *.sha256 + cache-seed: name: Seed cache (${{ matrix.arch }}) # Keep release-branch Windows validation advisory while the lane matures. @@ -109,7 +234,7 @@ jobs: needs: pr_metadata if: >- needs.pr_metadata.outputs.should_run == 'true' && - github.ref == 'refs/heads/windows' + (github.ref == 'refs/heads/windows' || github.event_name == 'workflow_dispatch') strategy: fail-fast: false matrix: @@ -124,11 +249,16 @@ jobs: env: READ_ONLY_GITHUB_TOKEN: ${{ github.token }} SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: windows-msvc-${{ matrix.arch }} + SCCACHE_GHA_VERSION: windows-msvc${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '-static-v1' || '' }}-${{ matrix.arch }} SCCACHE_CLIENT_SIDE: "1" SCCACHE_BASEDIRS: ${{ github.workspace }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Cache pinned Z3 package + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .cache/windows-z3 + key: windows-z3-${{ matrix.rust_target }}-${{ hashFiles('tasks/windows-z3.json') }} - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 with: install: false @@ -150,10 +280,16 @@ jobs: env: RUSTC_WRAPPER: "" with: - shared-key: windows-msvc-${{ matrix.arch }} + shared-key: windows-msvc${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '-static-v3' || '' }}-${{ matrix.arch }} cache-targets: "true" cache-on-failure: "true" cache-bin: "false" + - name: Test Windows dependency helpers + run: | + powershell -NoProfile -File tasks/scripts/windows-runtime-tests.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + powershell -NoProfile -File tasks/scripts/windows-z3-tests.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Lint Windows-supported crates # Keep the seed workload identical to the pull-request workload so the # windows-branch cache reflects the same lint and test commands. @@ -182,11 +318,16 @@ jobs: env: READ_ONLY_GITHUB_TOKEN: ${{ github.token }} SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: windows-msvc-${{ matrix.arch }} + SCCACHE_GHA_VERSION: windows-msvc${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '-static-v1' || '' }}-${{ matrix.arch }} SCCACHE_CLIENT_SIDE: "1" SCCACHE_BASEDIRS: ${{ github.workspace }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Cache pinned Z3 package + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: .cache/windows-z3 + key: windows-z3-${{ matrix.rust_target }}-${{ hashFiles('tasks/windows-z3.json') }} - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 with: install: false @@ -205,7 +346,7 @@ jobs: env: RUSTC_WRAPPER: "" with: - shared-key: windows-msvc-${{ matrix.arch }} + shared-key: windows-msvc${{ (github.event_name != 'workflow_dispatch' || inputs.static_runtime) && '-static-v3' || '' }}-${{ matrix.arch }} cache-targets: "true" cache-on-failure: "true" cache-bin: "false" diff --git a/CI.md b/CI.md index 8f039d34f9..2b1451b88b 100644 --- a/CI.md +++ b/CI.md @@ -79,6 +79,10 @@ runs native x64 and ARM64 lanes. Windows PR checks are opt-in: add `test:windows`, then select **Re-run all jobs** on the current Windows MSVC run. Subsequent mirrored commits run them automatically. Windows checks are not required for merging and do not run in merge queues. +Normal Windows CI retains pinned prebuilt Z3 and its original lint/test workload. +Manual dispatch exposes `static_runtime` (default `false`) to validate static +CRT/Z3 builds on both native runner architectures. This opt-in can build Z3 +from source; it uses separate compiler/target cache keys. Pushes to `windows` and manual runs on that branch seed the lint/test caches and run dependent release binary builds. Those jobs use `continue-on-error: true`, so inspect their individual results rather than treating a green workflow as qualification. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15402ccffd..dcc5c9170b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,19 +206,41 @@ cargo build -p openshell-prover-cli --features vendored-z3 Local gateway image and E2E builds enable `vendored-z3` so their copied gateway binaries do not need a shared Z3 library in the runtime image. -For x86-64 and ARM64 Windows MSVC builds from the `windows` branch, use one of these Z3 paths: - -- Prebuilt Z3 (the default for `windows:*` tasks): `z3-sys` downloads the - pinned Z3 5.1.0 GitHub release for the target architecture on the first - build. Cargo reuses the extracted archive from its target directory. Windows - CI authenticates the GitHub API request with `READ_ONLY_GITHUB_TOKEN` and - preserves the archive in the architecture-specific Cargo target cache. For - cold local builds, you may set `READ_ONLY_GITHUB_TOKEN` to avoid anonymous - GitHub API rate limits. -- System Z3: point `Z3_LIBRARY_PATH_OVERRIDE` at the directory containing the - target-compatible MSVC Z3 library and `Z3_SYS_Z3_HEADER` at the full path to `z3.h`. - The `windows:*` tasks use this path automatically when `Z3_LIBRARY_PATH_OVERRIDE` - is set. +Windows MSVC tasks on the `windows` branch use static CRT and Z3 by default. +They download the architecture-specific Z3 5.1.0 static `/MT` package from +`ghcr.io/nvidia/openshell/z3`, pinned by OCI manifest, ZIP, and build-manifest SHA256 digests in +`tasks/windows-z3.json`. Packages use MSVC 14.44 for compatibility with VS2022; +consumers need MSVC 14.44 or newer. The cache lives in `.cache/windows-z3`, outside Cargo +pruning. GitHub CI caches it separately from Rust outputs. Native PowerShell +downloads the package over HTTPS; Docker, ORAS, and GitHub credentials are not +required on build machines. + +If the package cannot be downloaded, the wrapper builds checksum-pinned 5.1.0 +source with CMake and Ninja. Digest, architecture, or version mismatches fail the +build. To request source explicitly, set `OPENSHELL_WINDOWS_Z3_SOURCE=1`; +`Z3_SRC_SOURCE_DIR` can supply a local 5.1.0 source tree. Source builds retain +native Release optimization in development and test builds. + +A supplied static `/MT` `libz3.lib` and matching header can be selected with +`Z3_LIBRARY_PATH_OVERRIDE` and `Z3_SYS_Z3_HEADER`. Explicit source selection +takes precedence over those overrides; otherwise a library override takes +precedence over a source directory. Each Cargo target resolves its own package. + +For compatibility with shared builds, set `OPENSHELL_WINDOWS_STATIC_RUNTIME=0`. +That mode retains the upstream pinned shared-prebuilt/system-library behavior +and stages the matching DLL. In default static mode, a legacy override containing +`libz3.dll` prints a migration warning and selects the pinned static package. +Static builds audit normal and delayed PE imports for all four executables, +reject Z3/redistributable CRT DLL dependencies, and remove stale staged Z3 DLLs. +The wrapper applies `.cargo/windows-static.toml`; direct Cargo commands retain +their existing configuration. + +Maintainers can rebuild the dependency with `windows:z3:package:x64` or +`windows:z3:package:arm64`. The Windows MSVC workflow's `publish_z3` dispatch +input builds and validates both packages, then publishes versioned OCI tags. +This separate mode does not run during normal PR or branch CI. It refuses to +replace existing tags. After publication, update the digests in +`tasks/windows-z3.json` and verify public downloads before updating the pin. `openshell-prover` itself has no `bindgen`/`libclang` dependency, so building just this crate does not require `LIBCLANG_PATH`: @@ -233,7 +255,7 @@ Run these commands in a `windows`-based checkout. To build the full set of Windows binaries, including `openshell-gateway.exe`, `openshell.exe`, `openshell-supervisor.exe`, and `openshell-windows-sandbox.exe`, use the `windows:build:x64` mise task instead of a -single-crate `cargo build`. It downloads the pinned prebuilt Z3 release by default. A +single-crate `cargo build`. It downloads the pinned static Z3 package by default. A full build also compiles crates that use `bindgen` (e.g. the MXC driver on Windows), so it requires `libclang.dll`; if LLVM is not on the default search path, set `LIBCLANG_PATH` to the directory containing `libclang.dll`: @@ -244,17 +266,17 @@ mise run --skip-tools windows:build:x64 ``` For ARM64, use `windows:build:arm64` with ARM64 MSVC components and host-native -LLVM. Keep the matching `libz3.dll` staged by the build beside the executables. +LLVM. The default binaries embed Z3 and the CRT. See [Windows validation](https://github.com/NVIDIA/OpenShell/blob/windows/TESTING.md#native-windows-validation) for native tests and MXC E2E requirements. Docker is not required for the native Windows build or MXC runtime. -To use a local x64 Z3 release instead of the prebuilt download, set +To use a local x64 static `/MT` Z3 package instead of the download, set `Z3_LIBRARY_PATH_OVERRIDE` and `Z3_SYS_Z3_HEADER` before running the task: ```powershell -$env:Z3_LIBRARY_PATH_OVERRIDE='C:\path\to\z3-5.1.0-x64-win\bin' -$env:Z3_SYS_Z3_HEADER='C:\path\to\z3-5.1.0-x64-win\include\z3.h' +$env:Z3_LIBRARY_PATH_OVERRIDE='C:\path\to\z3-5.1.0-static\lib' +$env:Z3_SYS_Z3_HEADER='C:\path\to\z3-5.1.0-static\include\z3.h' mise run --skip-tools windows:build:x64 ``` diff --git a/WINDOWS.md b/WINDOWS.md index cbd5be8b03..de5f4a1f60 100644 --- a/WINDOWS.md +++ b/WINDOWS.md @@ -73,14 +73,20 @@ mise run --skip-tools windows:build:arm64 The tasks produce `openshell.exe`, `openshell-gateway.exe`, `openshell-supervisor.exe`, and `openshell-windows-sandbox.exe` under `target//release/`. Keep these binaries from the same build -together with the matching `libz3.dll` staged by the build task. The target -triples are `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`. -`CARGO_TARGET_DIR` can override the output root. -The runtime host also needs the matching Visual C++ runtime. - -The build uses pinned prebuilt Z3 by default. Compile and test hosts do not -need a working MXC runtime; running native sandboxes does. These tasks do not -install an MSI, register a Windows service, or publish a release. +together. The target triples are `x86_64-pc-windows-msvc` and +`aarch64-pc-windows-msvc`. `CARGO_TARGET_DIR` can override the output root. + +Windows tasks embed the CRT in all four executables and Z3 in the gateway by +default. They reuse the pinned Z3 5.1.0 static `/MT` package from +`ghcr.io/nvidia/openshell/z3`, verify its digests, and cache it locally. Downloads +use HTTPS; Docker and a GHCR login are not required. Z3 builds from source only +when its package is unavailable or source is explicitly requested. These +executables do not need `libz3.dll` or the Visual C++ redistributable. +See [Windows build options](CONTRIBUTING.md#windows-full-build) for overrides. + +Compile and test hosts do not need a working MXC runtime; running native +sandboxes does. These tasks do not install an MSI, register a Windows service, +or publish a release. ## Prepare and run MXC diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 7a4c834af0..1fb9db7c7c 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -256,11 +256,13 @@ drivers need no UI-specific handling. UI policy itself does not require the ## Validation -Windows release build tasks stage the pinned, target-matching `libz3.dll` beside -the gateway and verify its architecture and copied SHA256. Conflicting cached -DLLs fail the build instead of selecting an arbitrary runtime. Local E2E requires -this adjacent DLL; rebuild for the native architecture if preflight reports it -missing. +Windows build tasks embed the CRT in all four executables and Z3 in the +gateway by default. They download and cache the pinned Z3 5.1.0 static `/MT` +package from GHCR, with source fallback when unavailable. Release builds reject +Z3 or redistributable CRT DLL imports. The resulting executables need neither +`libz3.dll` nor the Visual C++ redistributable. See +[Windows build options](../../CONTRIBUTING.md#windows-full-build) for explicit +source, library, and legacy shared-mode overrides. ### Remote Windows host @@ -284,11 +286,11 @@ architecture before testing. `-BuildDirectory` selects a dedicated Cargo cache. `-SkipBuild` reuses existing artifacts without claiming they match current sources. `-Scenario` selects an existing harness scenario. -The package includes `libz3.dll` from the target-specific Cargo build cache. -Windows prebuilt Z3 uses a dynamic runtime DLL. Supply `-Z3DllPath` if multiple -different cached versions exist or when using a custom Z3 build. The remote -host needs the native Visual C++ runtime; the runner validates DLL architecture -and gateway startup before invoking E2E. +The remote helper reads gateway imports and includes `libz3.dll` only for +shared builds. `-Z3DllPath` can select a matching DLL for those builds; their +runtime host also needs the native Visual C++ runtime. The helper checks DLL +architecture and gateway startup before E2E. Default static builds omit the DLL. +The separate MXC executor retains its own prerequisites. The example gateway explicitly enables caller driver config so tests can submit their workload command and directory. MXC does not implement external-resource diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 0b5709d7d4..2c10aeafb4 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -682,9 +682,6 @@ try { if (-not (Test-Path $policyDir)) { throw "e2e-policies/ directory not found at $policyDir" } - if (-not (Test-Path -LiteralPath (Join-Path $BinaryDir 'libz3.dll') -PathType Leaf)) { - throw "Missing libz3.dll beside the gateway. Re-run windows:build for this architecture, or package the matching Z3 runtime with these binaries." - } # Capture the pristine TOML once; every scenario renders a fresh copy from this. $tomlBase = Get-Content $tomlTemplate -Raw diff --git a/crates/openshell-driver-mxc/tests/windows_demo_examples.rs b/crates/openshell-driver-mxc/tests/windows_demo_examples.rs index 59f7cefe91..061b348b9f 100644 --- a/crates/openshell-driver-mxc/tests/windows_demo_examples.rs +++ b/crates/openshell-driver-mxc/tests/windows_demo_examples.rs @@ -261,6 +261,21 @@ if (Test-Path -LiteralPath $cliStateRoot) { throw "temporary state not removed" } } +#[test] +fn windows_static_runtime_and_dependency_selection() { + let output = Command::new("powershell.exe") + .args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-File"]) + .arg(examples().join("../../../tasks/scripts/windows-runtime-tests.ps1")) + .output() + .unwrap(); + assert!( + output.status.success(), + "Windows dependency/runtime regression failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] fn windows_build_stages_only_unambiguous_target_matching_z3() { // Synthetic PE headers exercise packaging checks, not MXC containment. diff --git a/tasks/scripts/run-mxc-remote.ps1 b/tasks/scripts/run-mxc-remote.ps1 index 8ead81e5c3..2fc0b6c78d 100644 --- a/tasks/scripts/run-mxc-remote.ps1 +++ b/tasks/scripts/run-mxc-remote.ps1 @@ -30,7 +30,9 @@ try { $native = Get-ItemPropertyValue 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' PROCESSOR_ARCHITECTURE $expected = if ($manifest.target -eq 'aarch64-pc-windows-msvc') { 0xaa64 } else { 0x8664 } if (($expected -eq 0xaa64 -and $native -ne 'ARM64') -or ($expected -eq 0x8664 -and $native -ne 'AMD64')) { throw 'Native host architecture changed.' } - foreach ($name in @('openshell.exe', 'openshell-gateway.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe', 'mxc-forwarding-agent.exe', 'libz3.dll')) { + $names = @('openshell.exe', 'openshell-gateway.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe', 'mxc-forwarding-agent.exe') + if (@($manifest.files | Where-Object { $_.path -eq 'bin/libz3.dll' }).Count) { $names += 'libz3.dll' } + foreach ($name in $names) { if ((Get-PeMachine (Join-Path $repo "bin/$name")) -ne $expected) { throw "Wrong executable architecture: $name" } } $wxc = $manifest.wxcExecPath diff --git a/tasks/scripts/test-mxc-remote.ps1 b/tasks/scripts/test-mxc-remote.ps1 index 6c842f6c31..d66f9b344a 100644 --- a/tasks/scripts/test-mxc-remote.ps1 +++ b/tasks/scripts/test-mxc-remote.ps1 @@ -23,6 +23,7 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $false $repo = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +. (Join-Path $PSScriptRoot 'windows-runtime.ps1') if ($HostName -notmatch '^[A-Za-z0-9.-]+$' -or $UserName -notmatch '^[A-Za-z0-9_.-]+$') { throw 'Use a hostname/IP and a simple SSH username (or configure an SSH alias).' } @@ -123,7 +124,9 @@ try { } $examples = 'crates/openshell-driver-mxc/examples' $files += @{ source = (Join-Path $BuildDirectory "$target/release/examples/mxc-forwarding-agent.exe"); path = 'bin/mxc-forwarding-agent.exe' } - $files += @{ source = (Find-Z3Runtime $BuildDirectory $target $Z3DllPath); path = 'bin/libz3.dll' } + if (Test-SharedWindowsZ3 (Join-Path $BuildDirectory "$target/release/openshell-gateway.exe")) { + $files += @{ source = (Find-Z3Runtime $BuildDirectory $target $Z3DllPath); path = 'bin/libz3.dll' } + } foreach ($path in @("$examples/run-mxc-e2e.ps1", "$examples/mxc-gateway.toml", 'tasks/scripts/run-mxc-remote.ps1')) { $files += @{ source = (Join-Path $repo $path); path = $path } } diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 500f8701e1..6fc49949ca 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -6,14 +6,18 @@ [CmdletBinding()] param( [Parameter(Mandatory = $true, Position = 0)] - [ValidateSet("check", "lint", "build", "build-mxc-fixtures", "test", "test-precommit", "test-unsupported", "test-mxc-real", "test-mxc-boundary", "test-mxc-loopback", "test-mxc-network", "test-mxc-audit", "artifacts", "ci")] + [ValidateSet("check", "lint", "build", "package-z3", "build-mxc-fixtures", "test", "test-precommit", "test-unsupported", "test-mxc-real", "test-mxc-boundary", "test-mxc-loopback", "test-mxc-network", "test-mxc-audit", "artifacts", "ci")] [string] $Action, [Parameter(Position = 1)] [ValidateSet("x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", "native", "all")] [string] $Target = "all", - [string] $LogDir + [string] $LogDir, + + [switch] $StaticRuntime = ($env:OPENSHELL_WINDOWS_STATIC_RUNTIME -ne '0'), + + [switch] $Z3Source = ($env:OPENSHELL_WINDOWS_Z3_SOURCE -eq '1') ) Set-StrictMode -Version Latest @@ -24,6 +28,8 @@ if (-not [System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform([Syst } $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +. (Join-Path $PSScriptRoot 'windows-runtime.ps1') +. (Join-Path $PSScriptRoot 'windows-z3.ps1') if (-not $LogDir) { $LogDir = $RepoRoot } @@ -36,6 +42,7 @@ $TargetDir = $env:CARGO_TARGET_DIR if ([string]::IsNullOrWhiteSpace($TargetDir)) { $TargetDir = Join-Path $RepoRoot "target" } +$TargetDir = [IO.Path]::GetFullPath($TargetDir) $BuildJobsValue = $env:OPENSHELL_WINDOWS_BUILD_JOBS if ([string]::IsNullOrWhiteSpace($BuildJobsValue)) { @@ -53,12 +60,16 @@ $WindowsCargoMutex = [System.Threading.Mutex]::new($false, "Local\OpenShellWindo $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-kubernetes-secrets --exclude openshell-driver-podman --exclude openshell-driver-vault --exclude openshell-driver-vm --exclude openshell-sandbox --exclude openshell-vfio" $WindowsClippyPackageExcludes = $UnsupportedDriverPackageExcludes $WindowsClippyLintArgs = "-D warnings -A dead-code -A unused-imports -A clippy::unused-async" -$PrebuiltZ3WorkspaceFeatures = "--features openshell-prover/prebuilt-z3" -$PrebuiltZ3ServerFeatures = "--features openshell-server/prebuilt-z3,openshell-prover/prebuilt-z3" -$PrebuiltZ3Version = "5.1.0" -$Z3WorkspaceFeatures = $PrebuiltZ3WorkspaceFeatures -$Z3ServerFeatures = $PrebuiltZ3ServerFeatures -$Z3GatewayFeatures = "--features prebuilt-z3" +$Z3Pin = Get-Content -LiteralPath (Join-Path $RepoRoot 'tasks/windows-z3.json') -Raw | ConvertFrom-Json +$Z3Version = $Z3Pin.version +$Z3SourceSha256 = $Z3Pin.sourceSha256 +$PrebuiltZ3Version = $Z3Version +# Each Cargo invocation restores the caller's overrides before resolving its +# target. A preceding x64 build must never supply its library to ARM64. +$UserZ3Environment = @{} +foreach ($name in @('Z3_LIBRARY_PATH_OVERRIDE', 'Z3_SYS_Z3_HEADER', 'Z3_SRC_SOURCE_DIR', 'Z3_SYS_Z3_VERSION')) { + $UserZ3Environment[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') +} function Get-VsInstallRoots { $programFiles = @( @@ -294,7 +305,7 @@ function Configure-Arm64CrossBuild([string[]] $RustTargets) { Write-Host "==> ARM64 cross-build toolchain" Write-Host " clang-cl: $clangCl" Write-Host " ninja: $ninja" - Write-Host " Z3: MSVC cl.exe with the Visual Studio generator" + Write-Host " Z3: pinned prebuilt, or MSVC cl.exe with Ninja for source builds" } function Get-HostArch { @@ -361,14 +372,118 @@ function Resolve-Z3HeaderPath([string] $HeaderPath) { return (Resolve-Path $HeaderPath).Path } -function Configure-Z3 { +function Resolve-Z3Source { + if ($env:Z3_SRC_SOURCE_DIR) { + $source = (Resolve-Path -LiteralPath $env:Z3_SRC_SOURCE_DIR).Path + } else { + # rust-cache prunes non-Cargo files under target, including VERSION.txt. + # Keep the complete source tree outside that cache's ownership. + $sourceRoot = Join-Path $RepoRoot '.cache/windows-z3-source' + $source = Join-Path $sourceRoot "z3-z3-$Z3Version" + if (-not (Test-Path -LiteralPath $source -PathType Container)) { + $staging = Join-Path $sourceRoot ("download-" + [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $staging -Force | Out-Null + try { + $archive = Join-Path $staging 'source.zip' + # Keep the existing Windows solver version. The z3-src crate's + # bundled source can advance independently of this release pin. + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/Z3Prover/z3/archive/refs/tags/z3-$Z3Version.zip" -OutFile $archive + if ((Get-Sha256 $archive) -ne $Z3SourceSha256) { throw 'Z3 source archive checksum mismatch.' } + # Hosted ARM64 runners can expose a PowerShell 7 archive module + # to Windows PowerShell 5.1. Use the .NET API shared by both. + Add-Type -AssemblyName System.IO.Compression.FileSystem + [IO.Compression.ZipFile]::ExtractToDirectory($archive, $staging) + New-Item -ItemType Directory -Path (Split-Path $source) -Force | Out-Null + Move-Item -LiteralPath (Join-Path $staging "z3-z3-$Z3Version") -Destination $source + } finally { Remove-Item -LiteralPath $staging -Recurse -Force } + } + } + $versionFile = Join-Path $source 'scripts/VERSION.txt' + if (-not (Test-Path -LiteralPath $versionFile) -or ([version](Get-Content -LiteralPath $versionFile -Raw).Trim()).ToString(3) -ne $Z3Version) { + throw "Z3 source must be version $Z3Version at $source." + } + return $source +} + +function Test-PinnedZ3Available([string] $RustTarget) { + foreach ($profile in @('debug', 'release')) { + $cached = Join-Path $TargetDir "$RustTarget/$profile/build/z3-sys-*/out/z3-$Z3Version/bin/libz3.lib" + if (Get-Item -Path $cached -ErrorAction SilentlyContinue) { return $true } + } + $arch = switch ($RustTarget) { + 'aarch64-pc-windows-msvc' { 'arm64' } + 'x86_64-pc-windows-msvc' { 'x64' } + default { return $false } + } + # z3-sys still owns downloading and caching the release. Only probe a cold + # target so a genuinely missing release asset can select the source fallback. + $url = "https://github.com/Z3Prover/z3/releases/download/z3-$Z3Version/z3-$Z3Version-$arch-win.zip" + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -UseBasicParsing -Method Head -Uri $url -TimeoutSec 30 | Out-Null + return $true + } catch { + $response = $_.Exception.PSObject.Properties['Response'] + if ($response -and $response.Value -and [int]$response.Value.StatusCode -eq 404) { return $false } + # Keep the original z3-sys download/error handling for inconclusive + # probes. HEAD may be blocked even when GET works, and commands whose + # Cargo graph does not use Z3 must still work offline. + return $true + } +} + +function Configure-Z3([string[]] $RustTargets) { + $sourceRequested = $Z3Source -or ([string]::IsNullOrWhiteSpace($env:Z3_LIBRARY_PATH_OVERRIDE) -and -not [string]::IsNullOrWhiteSpace($env:Z3_SRC_SOURCE_DIR)) + if ($sourceRequested) { + $env:Z3_LIBRARY_PATH_OVERRIDE = $null + $env:Z3_SYS_Z3_HEADER = $null + } + if ($StaticRuntime -and $env:Z3_LIBRARY_PATH_OVERRIDE -and (Test-Path -LiteralPath (Join-Path $env:Z3_LIBRARY_PATH_OVERRIDE 'libz3.dll'))) { + Write-Warning 'Ignoring the legacy shared Z3 override for the default static build. Set OPENSHELL_WINDOWS_STATIC_RUNTIME=0 to use shared CRT/Z3 explicitly.' + $env:Z3_LIBRARY_PATH_OVERRIDE = $null + $env:Z3_SYS_Z3_HEADER = $null + } if ([string]::IsNullOrWhiteSpace($env:Z3_LIBRARY_PATH_OVERRIDE)) { - Write-Host "==> Z3: prebuilt release" - $env:Z3_SYS_Z3_VERSION = $PrebuiltZ3Version - Write-Host " Z3_SYS_Z3_VERSION=$env:Z3_SYS_Z3_VERSION" - return [pscustomobject]@{ - WorkspaceFeatures = $PrebuiltZ3WorkspaceFeatures - ServerFeatures = $PrebuiltZ3ServerFeatures + $useSource = $sourceRequested + if ($StaticRuntime -and -not $useSource) { + if ($RustTargets.Count -ne 1) { throw 'Resolve static Z3 separately for each Cargo target.' } + $package = Resolve-Z3Package $RustTargets[0] + if ($package) { + $env:Z3_LIBRARY_PATH_OVERRIDE = Join-Path $package 'lib' + $env:Z3_SYS_Z3_HEADER = Join-Path $package 'include/z3.h' + $env:Z3_SYS_Z3_VERSION = $Z3Version + } else { $useSource = $true } + } + if (-not $StaticRuntime -and -not $useSource) { + foreach ($rustTarget in $RustTargets) { + if (-not (Test-PinnedZ3Available $rustTarget)) { + Write-Warning "Pinned Z3 $Z3Version prebuilt unavailable for $rustTarget; building pinned source." + $useSource = $true + } + } + } + if (-not $useSource -and -not $env:Z3_LIBRARY_PATH_OVERRIDE) { + $env:Z3_SYS_Z3_VERSION = $Z3Version + Write-Host "==> Z3: pinned prebuilt $Z3Version (z3-sys download/cache)" + return [pscustomobject]@{ + WorkspaceFeatures = '--features openshell-prover/prebuilt-z3' + ServerFeatures = '--features openshell-server/prebuilt-z3,openshell-prover/prebuilt-z3' + GatewayFeatures = '--features prebuilt-z3' + FromSource = $false + } + } + if ($useSource) { + $env:Z3_SRC_SOURCE_DIR = Resolve-Z3Source + $env:Z3_SYS_Z3_VERSION = $Z3Version + $env:CMAKE_BUILD_PARALLEL_LEVEL = "$WindowsBuildJobs" + Write-Host "==> Z3: source $env:Z3_SRC_SOURCE_DIR (static CRT: $StaticRuntime)" + return [pscustomobject]@{ + WorkspaceFeatures = '--features openshell-prover/vendored-z3' + ServerFeatures = '--features openshell-server/vendored-z3,openshell-prover/vendored-z3' + GatewayFeatures = '--features vendored-z3' + FromSource = $true + } } } @@ -384,28 +499,165 @@ function Configure-Z3 { $env:Z3_LIBRARY_PATH_OVERRIDE = $libDir $env:Z3_SYS_Z3_HEADER = Resolve-Z3HeaderPath $env:Z3_SYS_Z3_HEADER + if ($StaticRuntime -and (Test-Path -LiteralPath (Join-Path $libDir 'libz3.dll'))) { + throw 'StaticRuntime requires a genuine static libz3.lib built with /MT. Remove the shared override to build pinned source, or supply a static library.' + } if (($env:PATH -split ";") -notcontains $libDir) { $env:PATH = "$libDir;$env:PATH" } - Write-Host "==> Z3: system" + Write-Host "==> Z3: supplied MSVC library (static packages require /MT)" Write-Host " Z3_LIBRARY_PATH_OVERRIDE=$env:Z3_LIBRARY_PATH_OVERRIDE" Write-Host " Z3_SYS_Z3_HEADER=$env:Z3_SYS_Z3_HEADER" return [pscustomobject]@{ WorkspaceFeatures = "" ServerFeatures = "" + GatewayFeatures = "" + FromSource = $false } } +function Reset-WindowsCMakeGeneratorCache([string] $RustTarget) { + foreach ($profile in @('debug', 'release')) { + $buildRoot = Join-Path $TargetDir "$RustTarget/$profile/build" + if (-not (Test-Path -LiteralPath $buildRoot)) { continue } + $stale = @(Get-ChildItem -LiteralPath $buildRoot -Filter 'z3-sys-*' -Directory | Where-Object { + $cache = Join-Path $_.FullName 'out/build/CMakeCache.txt' + if (Test-Path -LiteralPath $cache) { + Select-String -LiteralPath $cache -Pattern '^CMAKE_GENERATOR:INTERNAL=(?!Ninja$)' -Quiet + } else { + # A pruned cache can retain the archive without its build rules. + Test-Path -LiteralPath (Join-Path $_.FullName 'out/lib/libz3.lib') + } + }) + if ($stale.Count) { + # Cargo must invalidate its build-script fingerprint and the linked + # archive too; deleting only CMake files can reuse an old solver. + $cargoProfile = if ($profile -eq 'debug') { 'dev' } else { 'release' } + Write-Host "==> Rebuilding cached $cargoProfile Z3 with Ninja" + & cargo clean -p z3-sys --target $RustTarget --profile $cargoProfile --target-dir $TargetDir + if ($LASTEXITCODE -ne 0) { throw 'Could not invalidate the old Z3 build cache.' } + } + } +} + +function Invoke-Z3Package([string] $RustTarget) { + $source = Resolve-Z3Source + $vsDevCmd = Resolve-VsDevCmd $RustTarget + $ninja = Resolve-NinjaPath + $cmake = Get-Command cmake.exe -ErrorAction SilentlyContinue + $cmakePath = if ($cmake) { $cmake.Source } else { Join-Path (Split-Path (Split-Path $ninja)) 'CMake/bin/cmake.exe' } + if (-not (Test-Path -LiteralPath $cmakePath)) { throw 'Install Visual Studio CMake tools to package Z3.' } + $build = Join-Path $TargetDir "z3-build/$Z3Version-msvc$($Z3Pin.msvcToolset)-v$($Z3Pin.packageRevision)/$RustTarget" + $output = Join-Path $TargetDir 'windows-z3-packages' + $install = Join-Path $build 'install' + $package = Join-Path $build 'package' + $toolsetRecord = Join-Path $build 'msvc-toolset.txt' + New-Item -ItemType Directory -Path $build, $output -Force | Out-Null + if (Test-Path -LiteralPath $package) { Remove-Item -LiteralPath $package -Recurse -Force } + $flags = @( + '-G Ninja', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_CXX_COMPILER=cl.exe', '-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded', + '-DZ3_BUILD_LIBZ3_SHARED=OFF', '-DZ3_BUILD_LIBZ3_MSVC_STATIC=ON', + '-DZ3_LINK_TIME_OPTIMIZATION=OFF', '-DZ3_BUILD_EXECUTABLE=ON', + '-DZ3_BUILD_TEST_EXECUTABLES=OFF', '-DZ3_ENABLE_EXAMPLE_TARGETS=OFF', + '-DZ3_BUILD_PYTHON_BINDINGS=OFF', '-DZ3_BUILD_DOTNET_BINDINGS=OFF', '-DZ3_BUILD_JAVA_BINDINGS=OFF' + ) + $log = Join-Path $LogDir "package-z3-$RustTarget.log" + $cmd = @( + "call `"$vsDevCmd`" -arch=$(Get-VsTargetArch $RustTarget) -host_arch=$(Get-HostArch) -vcvars_ver=$($Z3Pin.msvcToolset)", + "set VCToolsVersion>`"$toolsetRecord`"", + "findstr /i /b /l /c:`"VCToolsVersion=$($Z3Pin.msvcToolset).`" `"$toolsetRecord`"", + "`"$cmakePath`" -S `"$source`" -B `"$build`" $($flags -join ' ') -DCMAKE_MAKE_PROGRAM=`"$ninja`" -DCMAKE_INSTALL_PREFIX=`"$install`"", + "`"$cmakePath`" --build `"$build`" --target install --parallel $WindowsBuildJobs" + ) -join ' && ' + Write-Host "==> Building pinned Z3 $Z3Version /MT package for $RustTarget (one-time dependency build)" + & cmd /d /c "$cmd > `"$log`" 2>&1" + if ($LASTEXITCODE -ne 0) { + if (Test-Path -LiteralPath $log) { Get-Content -LiteralPath $log -Tail 100 } + throw "Z3 package build requires MSVC $($Z3Pin.msvcToolset), CMake, and Ninja. See $log" + } + $solver = Join-Path $install 'bin/z3.exe' + Assert-StaticWindowsRuntime $solver $RustTarget + $version = & $solver -version + if ($LASTEXITCODE -ne 0 -or $version -notmatch "^Z3 version $([regex]::Escape($Z3Version))( |$)") { throw "Wrong packaged Z3 version: $version" } + $query = Join-Path $build 'smoke.smt2' + Set-Content -LiteralPath $query -Encoding ASCII -Value '(declare-const x Int) (assert (> x 0)) (check-sat) (assert (< x 0)) (check-sat)' + $answers = @(& $solver $query) + if ($LASTEXITCODE -ne 0 -or ($answers -join ',') -ne 'sat,unsat') { throw "Z3 solver smoke failed: $answers" } + if (Select-String -LiteralPath (Join-Path $build 'build.ninja') -Pattern '(?i)(?:/GL|-flto)(?:\s|$)' -Quiet) { throw 'Portable Z3 archives must not contain compiler-specific LTO objects.' } + New-Item -ItemType Directory -Path $package -Force | Out-Null + Copy-Item -LiteralPath (Join-Path $install 'lib'), (Join-Path $install 'include') -Destination $package -Recurse + Copy-Item -LiteralPath (Join-Path $source 'LICENSE.txt') -Destination $package + $files = @(Get-ChildItem -LiteralPath $package -File -Recurse | ForEach-Object { + [ordered]@{ path = $_.FullName.Substring($package.Length + 1).Replace('\', '/'); sha256 = (Get-Sha256 $_.FullName).ToLowerInvariant(); size = $_.Length } + }) + [ordered]@{ + version = $Z3Version; target = $RustTarget; runtime = 'MT'; configuration = 'Release'; lto = $false + msvcToolset = (Get-Content -LiteralPath $toolsetRecord -Raw).Trim().Split('=')[1] + sourceUrl = "https://github.com/Z3Prover/z3/archive/refs/tags/z3-$Z3Version.zip" + sourceSha256 = $Z3SourceSha256; buildRevision = (& git -C $RepoRoot rev-parse HEAD) + cmakeFlags = $flags; solverSmoke = $answers; staticRuntimeAudit = 'passed'; files = $files + } | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $package 'build.json') -Encoding UTF8 + $archive = Join-Path $output "z3-$Z3Version-$RustTarget-mt-v$($Z3Pin.packageRevision).zip" + if (Test-Path -LiteralPath $archive) { Remove-Item -LiteralPath $archive } + Add-Type -AssemblyName System.IO.Compression.FileSystem + [IO.Compression.ZipFile]::CreateFromDirectory($package, $archive, [IO.Compression.CompressionLevel]::Optimal, $false) + "$(Get-Sha256 $archive) $(Split-Path -Leaf $archive)" | Set-Content -LiteralPath "$archive.sha256" -Encoding ASCII + Write-Host "==> Z3 package: $archive ($((Get-Item -LiteralPath $archive).Length) bytes)" +} + function Invoke-VsCargo { param( [Parameter(Mandatory = $true)] [string] $RustTarget, [Parameter(Mandatory = $true)] [string] $CargoArgs, - [Parameter(Mandatory = $true)] [string] $LogName + [Parameter(Mandatory = $true)] [string] $LogName, + [ValidateSet('Workspace', 'Server', 'Gateway', 'None')] [string] $Z3Scope = 'Workspace' ) + $Z3FromSource = $false + foreach ($name in $UserZ3Environment.Keys) { + [Environment]::SetEnvironmentVariable($name, $UserZ3Environment[$name], 'Process') + } + if ($Z3Scope -ne 'None') { + $z3 = Configure-Z3 @($RustTarget) + $Z3FromSource = $z3.FromSource + $features = $z3."${Z3Scope}Features" + if ($features) { + # Cargo feature options must precede test-harness arguments. + $separator = $CargoArgs.IndexOf(' -- ') + if ($separator -ge 0) { $CargoArgs = $CargoArgs.Insert($separator, " $features") } + else { $CargoArgs += " $features" } + } + } + # cmake-rs clears MSVC's configuration optimization flags when it selects + # the Visual Studio generator implicitly. Ninja preserves /O2 for Release + # Z3, including the dependency profile used by development/test builds. + if ($Z3FromSource) { + $ninja = Resolve-NinjaPath + Add-PathEntry (Split-Path -Parent $ninja) + } + + if (($Z3FromSource -or $StaticRuntime) -and $CargoArgs.StartsWith('cargo ')) { + # The old prebuilt solver was optimized even in debug/test builds. + # An unoptimized native Z3 can exhaust otherwise unchanged solver budgets. + $configs = @() + if ($Z3FromSource -and $CargoArgs -notmatch '--manifest-path') { + $configs += '--config "' + (Join-Path $RepoRoot '.cargo/windows-msvc.toml') + '"' + } + if ($StaticRuntime) { + $configs += '--config "' + (Join-Path $RepoRoot '.cargo/windows-static.toml') + '"' + } + $configArgs = $configs -join ' ' + if ($CargoArgs.StartsWith('cargo nextest run ')) { + # Nextest starts Cargo itself; global Cargo options are not forwarded. + $CargoArgs = 'cargo nextest run ' + $configArgs + ' ' + $CargoArgs.Substring(18) + } else { + $CargoArgs = 'cargo ' + $configArgs + ' ' + $CargoArgs.Substring(6) + } + } + & rustup target add $RustTarget if ($LASTEXITCODE -ne 0) { throw "rustup target add $RustTarget failed" @@ -430,6 +682,13 @@ function Invoke-VsCargo { "set `"CARGO_INCREMENTAL=0`"", "set `"RUSTC_WRAPPER=$rustcWrapper`"" ) + if ($Z3FromSource) { + $environmentSetup += @( + 'set "CMAKE_GENERATOR=Ninja"', + # z3-src otherwise adds MSBuild's -m argument to Ninja. + 'if not defined CMAKE_CXX_COMPILER_LAUNCHER (set "CMAKE_CXX_COMPILER_LAUNCHER=cmake;-E;env")' + ) + } if ($hostArch -eq "amd64" -and $RustTarget -eq "aarch64-pc-windows-msvc") { # Native ARM64 dependencies select their own compilers. Clear inherited # overrides so AWS-LC can select clang-cl inside its build script. @@ -463,6 +722,7 @@ function Invoke-VsCargo { throw "Timed out waiting for another Windows Cargo task to finish." } + if ($Z3FromSource) { Reset-WindowsCMakeGeneratorCache $RustTarget } $cmdWithLog = "$cmd > `"$logPath`" 2>&1" & cmd /v:on /d /c $cmdWithLog $exitCode = $LASTEXITCODE @@ -482,7 +742,7 @@ function Invoke-VsCargo { function Invoke-Check([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget $Z3WorkspaceFeatures" ` + -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget" ` -LogName "build-$RustTarget-check.log" Assert-GatewayExcludesUnsupportedDriverCrates $RustTarget } @@ -491,7 +751,7 @@ function Assert-GatewayExcludesUnsupportedDriverCrates([string] $RustTarget) { $logName = "build-$RustTarget-driver-tree.log" Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo tree -p openshell-gateway --target $RustTarget --prefix none" ` + -Z3Scope None -CargoArgs "cargo tree -p openshell-gateway --target $RustTarget --prefix none" ` -LogName $logName $logPath = Join-Path $LogDir $logName @@ -507,24 +767,35 @@ function Assert-GatewayExcludesUnsupportedDriverCrates([string] $RustTarget) { function Invoke-Lint([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo clippy --workspace --all-targets --no-deps $WindowsClippyPackageExcludes --target $RustTarget $Z3WorkspaceFeatures -- $WindowsClippyLintArgs" ` + -CargoArgs "cargo clippy --workspace --all-targets --no-deps $WindowsClippyPackageExcludes --target $RustTarget -- $WindowsClippyLintArgs" ` -LogName "lint-$RustTarget-workspace.log" Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo clippy --manifest-path e2e/rust/Cargo.toml --all-targets --no-deps --target $RustTarget -- $WindowsClippyLintArgs" ` + -Z3Scope None -CargoArgs "cargo clippy --manifest-path e2e/rust/Cargo.toml --all-targets --no-deps --target $RustTarget -- $WindowsClippyLintArgs" ` -LogName "lint-$RustTarget-e2e.log" Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo check --manifest-path examples/governance-interceptor/Cargo.toml --all-targets --target $RustTarget" ` + -Z3Scope None -CargoArgs "cargo check --manifest-path examples/governance-interceptor/Cargo.toml --all-targets --target $RustTarget" ` -LogName "lint-$RustTarget-governance-interceptor.log" } function Invoke-Build([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell --bin openshell-supervisor --bin openshell-windows-sandbox $Z3WorkspaceFeatures" ` + -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell --bin openshell-supervisor --bin openshell-windows-sandbox" ` -LogName "build-$RustTarget-release.log" - Stage-Z3Runtime $RustTarget + if ($StaticRuntime) { + foreach ($binary in @('openshell-gateway.exe', 'openshell.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe')) { + Assert-StaticWindowsRuntime (Join-Path $TargetDir "$RustTarget/release/$binary") $RustTarget + } + } + if (Test-SharedWindowsZ3 (Join-Path $TargetDir "$RustTarget/release/openshell-gateway.exe")) { + Stage-Z3Runtime $RustTarget + return + } + # Do not leave a previously staged DLL in a directory used for ZIP packages. + $staleDll = Join-Path $TargetDir "$RustTarget/release/libz3.dll" + if (Test-Path -LiteralPath $staleDll) { Remove-Item -LiteralPath $staleDll } } function Stage-Z3Runtime([string] $RustTarget) { @@ -563,7 +834,7 @@ function Invoke-Test([string] $RustTarget) { Assert-NativeTestTarget $RustTarget Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo nextest run --profile ci --workspace $UnsupportedDriverPackageExcludes --target $RustTarget $Z3WorkspaceFeatures" ` + -CargoArgs "cargo nextest run --profile ci --workspace $UnsupportedDriverPackageExcludes --target $RustTarget" ` -LogName "test-$RustTarget.log" } @@ -571,7 +842,7 @@ function Invoke-PreCommitTest([string] $RustTarget) { Assert-NativeTestTarget $RustTarget Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo nextest run --profile ci --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --features openshell-server/test-support $Z3ServerFeatures" ` + -Z3Scope Server -CargoArgs "cargo nextest run --profile ci --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --features openshell-server/test-support" ` -LogName "test-$RustTarget-precommit.log" } @@ -585,7 +856,7 @@ function Invoke-UnsupportedContractTests([string] $RustTarget) { foreach ($test in $tests) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test -p openshell-gateway --target $RustTarget $test $Z3GatewayFeatures" ` + -Z3Scope Gateway -CargoArgs "cargo test -p openshell-gateway --target $RustTarget $test" ` -LogName "test-$RustTarget-unsupported-$test.log" } @@ -594,7 +865,7 @@ function Invoke-UnsupportedContractTests([string] $RustTarget) { $variant = if ($features) { $features.Replace(",", "-") } else { "protocol-only" } Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test -p openshell-gateway --lib --target $RustTarget --no-default-features $featureArgs $Z3GatewayFeatures" ` + -Z3Scope Gateway -CargoArgs "cargo test -p openshell-gateway --lib --target $RustTarget --no-default-features $featureArgs" ` -LogName "test-$RustTarget-selective-$variant.log" } } @@ -603,7 +874,7 @@ function Invoke-MxcRealTests([string] $RustTarget) { Assert-NativeTestTarget $RustTarget Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test -p openshell-driver-mxc --test wxc_exec_real --target $RustTarget -- --ignored --test-threads=1 --nocapture" ` + -Z3Scope None -CargoArgs "cargo test -p openshell-driver-mxc --test wxc_exec_real --target $RustTarget -- --ignored --test-threads=1 --nocapture" ` -LogName "test-$RustTarget-mxc-real.log" } @@ -658,10 +929,6 @@ if ($Action -in @("test", "test-precommit", "test-unsupported", "test-mxc-real") } if ($Action -in @("check", "lint", "build", "build-mxc-fixtures", "test", "test-precommit", "test-unsupported", "test-mxc-real", "test-mxc-boundary", "test-mxc-loopback", "test-mxc-network", "test-mxc-audit", "ci")) { - $z3Features = Configure-Z3 - $Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures - $Z3ServerFeatures = $z3Features.ServerFeatures - $Z3GatewayFeatures = if ($z3Features.WorkspaceFeatures) { "--features prebuilt-z3" } else { "" } $env:LIBCLANG_PATH = Resolve-LibclangPath Add-PathEntry $env:LIBCLANG_PATH Write-Host "==> LIBCLANG_PATH=$env:LIBCLANG_PATH" @@ -669,6 +936,9 @@ if ($Action -in @("check", "lint", "build", "build-mxc-fixtures", "test", "test- } switch ($Action) { + "package-z3" { + foreach ($rustTarget in $targets) { Invoke-Z3Package $rustTarget } + } "check" { foreach ($rustTarget in $targets) { Invoke-Check $rustTarget @@ -688,7 +958,7 @@ switch ($Action) { "build-mxc-fixtures" { foreach ($rustTarget in $targets) { Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo build --release -p openshell-driver-mxc --example mxc-forwarding-agent --target $rustTarget" ` + -Z3Scope None -CargoArgs "cargo build --release -p openshell-driver-mxc --example mxc-forwarding-agent --target $rustTarget" ` -LogName "build-$rustTarget-mxc-fixtures.log" } } @@ -711,7 +981,7 @@ switch ($Action) { foreach ($rustTarget in $targets) { Assert-NativeTestTarget $rustTarget Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo test --release -p openshell-mxc-boundary --lib --target $rustTarget -- --test-threads=1" ` + -Z3Scope None -CargoArgs "cargo test --release -p openshell-mxc-boundary --lib --target $rustTarget -- --test-threads=1" ` -LogName "test-$rustTarget-mxc-boundary.log" } } @@ -719,7 +989,7 @@ switch ($Action) { foreach ($rustTarget in $targets) { Assert-NativeTestTarget $rustTarget Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo test --release -p openshell-mxc-boundary --lib --target $rustTarget native_same_container_loopback -- --ignored --nocapture --test-threads=1" ` + -Z3Scope None -CargoArgs "cargo test --release -p openshell-mxc-boundary --lib --target $rustTarget native_same_container_loopback -- --ignored --nocapture --test-threads=1" ` -LogName "test-$rustTarget-mxc-loopback.log" } } @@ -728,7 +998,7 @@ switch ($Action) { Assert-NativeTestTarget $rustTarget foreach ($testName in @('pc_https_egress_reads_injected_ca_bundle', 'pc_proxy_scopes_network_policy_to_socket_owner')) { Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo test --release -p openshell-driver-mxc --test wxc_exec_real --target $rustTarget $testName -- --ignored --exact --test-threads=1 --nocapture" ` + -Z3Scope None -CargoArgs "cargo test --release -p openshell-driver-mxc --test wxc_exec_real --target $rustTarget $testName -- --ignored --exact --test-threads=1 --nocapture" ` -LogName "test-$rustTarget-$testName.log" } } @@ -737,19 +1007,24 @@ switch ($Action) { foreach ($rustTarget in $targets) { Assert-NativeTestTarget $rustTarget Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo test --release -p openshell-driver-mxc --lib --target $rustTarget etw_consumer::tests -- --test-threads=1 --nocapture" ` + -Z3Scope None -CargoArgs "cargo test --release -p openshell-driver-mxc --lib --target $rustTarget etw_consumer::tests -- --test-threads=1 --nocapture" ` -LogName "test-$rustTarget-mxc-audit-attribution.log" Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo test --release -p openshell-server --lib --target $rustTarget $Z3ServerFeatures mxc_audit_example -- --nocapture" ` + -Z3Scope Server -CargoArgs "cargo test --release -p openshell-server --lib --target $rustTarget mxc_audit_example -- --nocapture" ` -LogName "test-$rustTarget-mxc-audit-config.log" Invoke-VsCargo -RustTarget $rustTarget ` - -CargoArgs "cargo test --release -p openshell-driver-mxc --test windows_demo_examples --target $rustTarget -- --test-threads=1 --nocapture" ` + -Z3Scope None -CargoArgs "cargo test --release -p openshell-driver-mxc --test windows_demo_examples --target $rustTarget -- --test-threads=1 --nocapture" ` -LogName "test-$rustTarget-mxc-audit-examples.log" $package = Join-Path $TargetDir "mxc-ocsf-qualification-$rustTarget" New-Item -ItemType Directory -Force $package | Out-Null - foreach ($binary in @('openshell-gateway.exe', 'openshell.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe', 'libz3.dll')) { + foreach ($binary in @('openshell-gateway.exe', 'openshell.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe')) { Copy-Item -LiteralPath (Join-Path $TargetDir "$rustTarget/release/$binary") -Destination $package -Force } + if (Test-SharedWindowsZ3 (Join-Path $package 'openshell-gateway.exe')) { + Copy-Item -LiteralPath (Join-Path $TargetDir "$rustTarget/release/libz3.dll") -Destination $package -Force + } elseif (Test-Path -LiteralPath (Join-Path $package 'libz3.dll')) { + Remove-Item -LiteralPath (Join-Path $package 'libz3.dll') + } foreach ($artifact in @('run-ocsf-audit.ps1', 'mxc-ocsf-audit.toml', 'ocsf-audit.yaml')) { Copy-Item -LiteralPath (Join-Path $RepoRoot "crates/openshell-driver-mxc/examples/$artifact") -Destination $package -Force } diff --git a/tasks/scripts/windows-runtime-tests.ps1 b/tasks/scripts/windows-runtime-tests.ps1 new file mode 100644 index 0000000000..fb8dc225e2 --- /dev/null +++ b/tasks/scripts/windows-runtime-tests.ps1 @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +. (Join-Path $PSScriptRoot 'windows-runtime.ps1') + +function New-Image([int] $Machine, [string] $Dependency, [switch] $Delay) { + $bytes = New-Object byte[] 1024 + function Set-U16([int] $Offset, [uint16] $Value) { [BitConverter]::GetBytes($Value).CopyTo($bytes, $Offset) } + function Set-U32([int] $Offset, [uint32] $Value) { [BitConverter]::GetBytes($Value).CopyTo($bytes, $Offset) } + Set-U16 0 0x5a4d + Set-U32 60 64 + Set-U32 64 0x4550 + Set-U16 68 $Machine + Set-U16 70 1 + Set-U16 84 240 + Set-U16 88 0x20b + Set-U32 196 16 + # One section maps RVAs 0x1000..0x11ff to file offsets 512..1023. + Set-U32 336 512 + Set-U32 340 0x1000 + Set-U32 344 512 + Set-U32 348 512 + if ($Dependency) { + if ($Delay) { + Set-U32 304 0x1000 + Set-U32 308 64 + Set-U32 512 1 + Set-U32 516 0x1080 + } else { + Set-U32 208 0x1000 + Set-U32 212 40 + Set-U32 524 0x1080 + } + [Text.Encoding]::ASCII.GetBytes($Dependency).CopyTo($bytes, 640) + } + return ,$bytes +} + +$fixture = [IO.Path]::GetTempFileName() +$count = 0 +try { + foreach ($entry in @(@(0xaa64, 'aarch64-pc-windows-msvc'), @(0x8664, 'x86_64-pc-windows-msvc'))) { + [IO.File]::WriteAllBytes($fixture, (New-Image $entry[0] 'KERNEL32.dll')) + Assert-StaticWindowsRuntime $fixture $entry[1] + $count++ + } + foreach ($dependency in @('libz3.dll', 'VCRUNTIME140.dll', 'VCRUNTIME140_1.dll', 'MSVCP140.dll', 'MSVCP140_ATOMIC_WAIT.dll', 'ucrtbase.dll', 'api-ms-win-crt-heap-l1-1-0.dll')) { + foreach ($delay in @($false, $true)) { + [IO.File]::WriteAllBytes($fixture, (New-Image 0xaa64 $dependency -Delay:$delay)) + $image = Get-WindowsImageDependencies $fixture + if ($image.Dependencies -notcontains $dependency) { throw "Import not detected: $dependency" } + if ((Test-SharedWindowsZ3 $fixture) -ne ($dependency -eq 'libz3.dll')) { throw 'Shared Z3 detection disagrees with imports.' } + $rejected = $false + try { Assert-StaticWindowsRuntime $fixture 'aarch64-pc-windows-msvc' } catch { + if ($_.Exception.Message -notmatch 'Dynamic Z3/CRT') { throw } + $rejected = $true + } + if (-not $rejected) { throw "Dynamic dependency accepted: $dependency" } + $count++ + } + } + [IO.File]::WriteAllBytes($fixture, (New-Image 0x8664 '')) + $rejected = $false + try { Assert-StaticWindowsRuntime $fixture 'aarch64-pc-windows-msvc' } catch { $rejected = $true } + if (-not $rejected) { throw 'Wrong PE architecture accepted.' } + $count++ + [IO.File]::WriteAllBytes($fixture, [byte[]]@(1, 2, 3, 4)) + $rejected = $false + try { Get-WindowsImageDependencies $fixture | Out-Null } catch { $rejected = $true } + if (-not $rejected) { throw 'Truncated PE image accepted.' } + $count++ +} finally { Remove-Item -LiteralPath $fixture } + +# Exercise prebuilt selection and source fallback without network access. +$tokens = $null; $errors = $null +$ast = [Management.Automation.Language.Parser]::ParseFile((Join-Path $PSScriptRoot 'windows-msvc.ps1'), [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw ($errors | Out-String) } +foreach ($name in @('Get-Sha256', 'Resolve-Z3HeaderPath', 'Resolve-Z3Source', 'Test-PinnedZ3Available', 'Configure-Z3', 'Reset-WindowsCMakeGeneratorCache')) { + $function = $ast.Find({ param($node) $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $name }, $true) + if ($null -eq $function) { throw "Missing build helper: $name" } + Invoke-Expression $function.Extent.Text +} +$sourceFixture = Join-Path ([IO.Path]::GetTempPath()) ('openshell-source-test-' + [guid]::NewGuid().ToString('N')) +try { + $scripts = Join-Path $sourceFixture 'input/z3-z3-5.1.0/scripts' + New-Item -ItemType Directory -Path $scripts -Force | Out-Null + Set-Content -LiteralPath (Join-Path $scripts 'VERSION.txt') -Value '5.1.0.0' -Encoding ASCII + Add-Type -AssemblyName System.IO.Compression.FileSystem + $sourceArchive = Join-Path $sourceFixture 'source.zip' + [IO.Compression.ZipFile]::CreateFromDirectory((Join-Path $sourceFixture 'input'), $sourceArchive) + Add-Type @' +public sealed class OpenShellZ3TestResponse { public int StatusCode; } +public sealed class OpenShellZ3TestException : System.Exception { + public OpenShellZ3TestResponse Response; + public OpenShellZ3TestException(int status) { Response = new OpenShellZ3TestResponse(); Response.StatusCode = status; } +} +'@ + $script:headStatus = 200 + $script:headCalls = @() + $script:sourceDownloads = 0 + function Invoke-WebRequest([switch] $UseBasicParsing, [string] $Uri, [string] $OutFile, [string] $Method, [int] $TimeoutSec) { + if ($Method -eq 'Head') { + $script:headCalls += $Uri + if ($script:headStatus -eq 0) { throw [IO.IOException]::new('Transport unavailable') } + if ($script:headStatus -ne 200) { throw [OpenShellZ3TestException]::new($script:headStatus) } + } else { + $script:sourceDownloads++ + Copy-Item -LiteralPath $sourceArchive -Destination $OutFile + } + } + $Z3Version = '5.1.0' + $Z3SourceSha256 = Get-Sha256 $sourceArchive + $WindowsBuildJobs = 2 + $RepoRoot = $sourceFixture + $TargetDir = Join-Path $sourceFixture 'target' + $oldBuild = Join-Path $TargetDir 'aarch64-pc-windows-msvc/release/build/z3-sys-old/out/build' + $newBuild = Join-Path $TargetDir 'aarch64-pc-windows-msvc/debug/build/z3-sys-new/out/build' + $otherTarget = Join-Path $TargetDir 'x86_64-pc-windows-msvc/release/build/z3-sys-old/out/build' + foreach ($build in @($oldBuild, $newBuild, $otherTarget)) { + New-Item -ItemType Directory -Path $build -Force | Out-Null + Set-Content -LiteralPath (Join-Path $build 'CMakeCache.txt') -Value 'CMAKE_GENERATOR:INTERNAL=Visual Studio 17 2022' + } + Set-Content -LiteralPath (Join-Path $newBuild 'CMakeCache.txt') -Value 'CMAKE_GENERATOR:INTERNAL=Ninja' + $script:cleanCalls = @() + function cargo { + $script:cleanCalls += ,@($args) + $global:LASTEXITCODE = 0 + } + Reset-WindowsCMakeGeneratorCache 'aarch64-pc-windows-msvc' + $expected = @('clean', '-p', 'z3-sys', '--target', 'aarch64-pc-windows-msvc', '--profile', 'release', '--target-dir', $TargetDir) + if ($script:cleanCalls.Count -ne 1 -or ($script:cleanCalls[0] -join '|') -ne ($expected -join '|')) { throw 'Cargo was not asked to invalidate precisely the stale solver profile and target.' } + if (-not (Test-Path -LiteralPath $newBuild) -or -not (Test-Path -LiteralPath $otherTarget)) { throw 'Unrelated build files removed.' } + $count++ + $archiveDir = Join-Path (Split-Path $oldBuild) 'lib' + New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null + Set-Content -LiteralPath (Join-Path $archiveDir 'libz3.lib') -Value 'old cached archive' + Remove-Item -LiteralPath (Join-Path $oldBuild 'CMakeCache.txt') + Reset-WindowsCMakeGeneratorCache 'aarch64-pc-windows-msvc' + if ($script:cleanCalls.Count -ne 2) { throw 'Cached solver archive without build rules was reused.' } + $count++ + + $script:staticPackage = $null + function Resolve-Z3Package([string] $RustTarget) { return $script:staticPackage } + $Z3Source = $false + $StaticRuntime = $false + $env:Z3_SRC_SOURCE_DIR = $null + $env:Z3_LIBRARY_PATH_OVERRIDE = $null + $env:Z3_SYS_Z3_HEADER = $null + $features = Configure-Z3 @('aarch64-pc-windows-msvc', 'x86_64-pc-windows-msvc') + if ($features.WorkspaceFeatures -notmatch 'prebuilt-z3' -or $features.FromSource -or $script:sourceDownloads) { throw 'Default no longer reuses pinned prebuilts.' } + if ($script:headCalls.Count -ne 2 -or $script:headCalls[0] -notmatch 'z3-5.1.0-arm64-win.zip$' -or $script:headCalls[1] -notmatch 'z3-5.1.0-x64-win.zip$') { throw 'Prebuilt architecture/version mapping changed.' } + $count++ + $cache = Join-Path $TargetDir 'aarch64-pc-windows-msvc/debug/build/z3-sys-cached/out/z3-5.1.0/bin' + New-Item -ItemType Directory -Force -Path $cache | Out-Null + Set-Content -LiteralPath (Join-Path $cache 'libz3.lib') -Value 'cached prebuilt' + $script:headStatus = 500 + if (-not (Test-PinnedZ3Available 'aarch64-pc-windows-msvc') -or $script:headCalls.Count -ne 2) { throw 'Cached prebuilt was not reused offline.' } + $count++ + $features = Configure-Z3 @('x86_64-pc-windows-msvc') + if ($features.FromSource -or $features.WorkspaceFeatures -notmatch 'prebuilt-z3' -or $script:sourceDownloads) { throw 'Inconclusive HEAD changed the original prebuilt path.' } + $count++ + $script:headStatus = 0 + if (-not (Test-PinnedZ3Available 'x86_64-pc-windows-msvc')) { throw 'Transport failure disabled the original downloader or offline non-Z3 commands.' } + $count++ + + $env:Z3_LIBRARY_PATH_OVERRIDE = $sourceFixture + Set-Content -LiteralPath (Join-Path $sourceFixture 'libz3.lib') -Value 'supplied library' + $env:Z3_SYS_Z3_HEADER = Join-Path $sourceFixture 'z3.h' + Set-Content -LiteralPath $env:Z3_SYS_Z3_HEADER -Value '// header' + Set-Content -LiteralPath (Join-Path $sourceFixture 'libz3.dll') -Value 'legacy shared runtime' + $features = Configure-Z3 @('aarch64-pc-windows-msvc') + if ($features.WorkspaceFeatures -or $features.FromSource -or $script:sourceDownloads) { throw 'Shared library override no longer works by default.' } + $count++ + $StaticRuntime = $true + $features = Configure-Z3 @('aarch64-pc-windows-msvc') + if (-not $features.FromSource -or $env:Z3_LIBRARY_PATH_OVERRIDE -or $script:sourceDownloads -ne 1) { throw 'Legacy shared override defeated the default static package/source selection.' } + $count++ + Remove-Item -LiteralPath (Join-Path $sourceFixture 'libz3.dll') + $env:Z3_LIBRARY_PATH_OVERRIDE = $sourceFixture + $env:Z3_SYS_Z3_HEADER = Join-Path $sourceFixture 'z3.h' + $env:Z3_SRC_SOURCE_DIR = Join-Path $sourceFixture 'source-must-not-be-used' + $features = Configure-Z3 @('aarch64-pc-windows-msvc') + if ($features.WorkspaceFeatures -or $features.FromSource -or $script:sourceDownloads -ne 1) { throw 'Supplied static library unexpectedly rebuilt source.' } + $count++ + $env:Z3_LIBRARY_PATH_OVERRIDE = $null + $env:Z3_SRC_SOURCE_DIR = $null + $script:staticPackage = Join-Path $sourceFixture 'ghcr-package' + New-Item -ItemType Directory -Path (Join-Path $script:staticPackage 'lib'), (Join-Path $script:staticPackage 'include') -Force | Out-Null + Copy-Item -LiteralPath (Join-Path $sourceFixture 'libz3.lib') -Destination (Join-Path $script:staticPackage 'lib/libz3.lib') + Copy-Item -LiteralPath (Join-Path $sourceFixture 'z3.h') -Destination (Join-Path $script:staticPackage 'include/z3.h') + $features = Configure-Z3 @('aarch64-pc-windows-msvc') + if ($features.WorkspaceFeatures -or $features.FromSource -or $env:Z3_SRC_SOURCE_DIR -or $script:sourceDownloads -ne 1) { throw 'Available static package unexpectedly selected source compilation.' } + $count++ + $script:staticPackage = $null + $env:Z3_LIBRARY_PATH_OVERRIDE = $null + $features = Configure-Z3 @('aarch64-pc-windows-msvc') + if ($features.WorkspaceFeatures -notmatch 'vendored-z3' -or -not $features.FromSource -or $script:sourceDownloads -ne 1) { throw 'Unavailable static package did not reuse pinned source.' } + $count++ + if ($env:Z3_SRC_SOURCE_DIR.StartsWith($TargetDir, [StringComparison]::OrdinalIgnoreCase)) { throw 'Source cache is inside the pruned Cargo target directory.' } + $count++ + if (-not (Test-Path -LiteralPath (Join-Path $env:Z3_SRC_SOURCE_DIR 'scripts/VERSION.txt'))) { throw 'Cold source extraction failed.' } + $count++ + $env:Z3_SRC_SOURCE_DIR = $null + $Z3SourceSha256 = '0' * 64 + # A verified cache can be reused without downloading another archive. + if (-not (Resolve-Z3Source)) { throw 'Source cache reuse failed.' } + $count++ + $StaticRuntime = $false + $script:headStatus = 404 + $env:Z3_SRC_SOURCE_DIR = $null + $features = Configure-Z3 @('x86_64-pc-windows-msvc') + if (-not $features.FromSource -or $script:sourceDownloads -ne 1) { throw 'Missing pinned prebuilt did not reuse source fallback.' } + $count++ + $env:Z3_SRC_SOURCE_DIR = $null + $Z3Source = $true + $script:headStatus = 500 + $features = Configure-Z3 @('x86_64-pc-windows-msvc') + if (-not $features.FromSource -or $script:sourceDownloads -ne 1) { throw 'Explicit source request queried a prebuilt or missed the cache.' } + $count++ + $env:Z3_SRC_SOURCE_DIR = $null + $RepoRoot = Join-Path $sourceFixture 'bad-checksum' + $rejected = $false + try { Resolve-Z3Source | Out-Null } catch { + if ($_.Exception.Message -notmatch 'checksum') { throw } + $rejected = $true + } + if (-not $rejected) { throw 'Unverified Z3 source archive accepted.' } + $count++ + $env:Z3_SRC_SOURCE_DIR = Split-Path $scripts + Set-Content -LiteralPath (Join-Path $scripts 'VERSION.txt') -Value '5.3.2.0' + $rejected = $false + try { Resolve-Z3Source | Out-Null } catch { $rejected = $true } + if (-not $rejected) { throw 'Wrong solver source version accepted.' } + $count++ +} finally { Remove-Item -LiteralPath $sourceFixture -Recurse -Force } +Write-Host "$count Windows dependency/runtime regression checks passed. No MXC qualification claimed." diff --git a/tasks/scripts/windows-runtime.ps1 b/tasks/scripts/windows-runtime.ps1 new file mode 100644 index 0000000000..7b969a4241 --- /dev/null +++ b/tasks/scripts/windows-runtime.ps1 @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Read PE imports without requiring dumpbin or a developer environment on the +# runtime host. Both supported Windows architectures use PE32+ images. +function Get-WindowsImageDependencies([string] $Path) { + $reader = [IO.BinaryReader]::new([IO.File]::OpenRead($Path)) + $stream = $reader.BaseStream + try { + if ($stream.Length -lt 64 -or $reader.ReadUInt16() -ne 0x5a4d) { throw "Invalid PE image: $Path" } + $stream.Position = 0x3c + $pe = $reader.ReadUInt32() + if ($pe -gt $stream.Length - 24) { throw "Invalid PE offset: $Path" } + $stream.Position = $pe + if ($reader.ReadUInt32() -ne 0x4550) { throw "Invalid PE signature: $Path" } + $machine = $reader.ReadUInt16() + $sectionCount = $reader.ReadUInt16() + $stream.Position = $pe + 20 + $optionalSize = $reader.ReadUInt16() + $optional = $pe + 24 + if ($optionalSize -lt 128 -or $optional + $optionalSize + 40 * $sectionCount -gt $stream.Length) { throw "Invalid PE headers: $Path" } + $stream.Position = $optional + if ($reader.ReadUInt16() -ne 0x20b) { throw "Expected PE32+ image: $Path" } + $stream.Position = $optional + 108 + $directoryCount = $reader.ReadUInt32() + if ($directoryCount -lt 2 -or 112 + 8 * $directoryCount -gt $optionalSize) { throw "Invalid PE directories: $Path" } + $sections = @() + for ($i = 0; $i -lt $sectionCount; $i++) { + $stream.Position = $optional + $optionalSize + 40 * $i + 8 + $sections += [pscustomobject]@{ + VirtualSize = $reader.ReadUInt32(); VirtualAddress = $reader.ReadUInt32() + RawSize = $reader.ReadUInt32(); RawOffset = $reader.ReadUInt32() + } + } + function Resolve-ImageRva([uint32] $Rva) { + foreach ($section in $sections) { + $relative = [long]$Rva - $section.VirtualAddress + if ($relative -ge 0 -and $relative -lt $section.RawSize) { + $offset = [long]$section.RawOffset + $relative + if ($offset -lt $stream.Length) { return $offset } + } + } + throw "Invalid PE RVA in $Path" + } + $dependencies = @() + foreach ($index in @(1, 13)) { + if ($directoryCount -le $index) { continue } + $stream.Position = $optional + 112 + 8 * $index + $rva = $reader.ReadUInt32() + $size = $reader.ReadUInt32() + if ($rva -eq 0) { continue } + $descriptorSize = if ($index -eq 1) { 20 } else { 32 } + $directory = Resolve-ImageRva $rva + $terminated = $false + for ($position = $directory; $position + $descriptorSize -le [Math]::Min($stream.Length, $directory + $size); $position += $descriptorSize) { + $stream.Position = $position + $fields = @(for ($j = 0; $j -lt $descriptorSize / 4; $j++) { $reader.ReadUInt32() }) + if (@($fields | Where-Object { $_ -ne 0 }).Count -eq 0) { $terminated = $true; break } + # Modern delay-import descriptors store RVAs (attribute bit 0). + if ($index -eq 13 -and ($fields[0] -band 1) -eq 0) { throw "Unsupported delay-import addressing: $Path" } + $nameRva = if ($index -eq 1) { $fields[3] } else { $fields[1] } + $stream.Position = Resolve-ImageRva $nameRva + $name = [Text.StringBuilder]::new() + do { + if ($stream.Position -ge $stream.Length -or $name.Length -gt 260) { throw "Invalid PE import name: $Path" } + $byte = $reader.ReadByte() + if ($byte) { [void]$name.Append([char]$byte) } + } while ($byte) + $dependencies += $name.ToString() + } + if (-not $terminated) { throw "Unterminated PE import directory: $Path" } + } + return [pscustomobject]@{ Machine = $machine; Dependencies = @($dependencies | Sort-Object -Unique) } + } finally { $reader.Dispose() } +} + +function Assert-StaticWindowsRuntime([string] $Path, [string] $RustTarget) { + $image = Get-WindowsImageDependencies $Path + $expected = switch ($RustTarget) { + 'aarch64-pc-windows-msvc' { 0xaa64 } + 'x86_64-pc-windows-msvc' { 0x8664 } + default { throw "Unsupported Windows target: $RustTarget" } + } + if ($image.Machine -ne $expected) { throw "PE architecture does not match ${RustTarget}: $Path" } + $dynamic = @($image.Dependencies | Where-Object { + $_ -match '^(libz3|z3|vcruntime\d+[^.]*|msvcp\d+[^.]*|msvcr\d+[^.]*|concrt\d+[^.]*|ucrtbased?|api-ms-win-crt-[^.]+)\.dll$' + }) + if ($dynamic.Count) { throw "Dynamic Z3/CRT dependency in ${Path}: $($dynamic -join ', ')" } + Write-Host "==> Static Z3/CRT and PE architecture verified: $Path" +} + +function Test-SharedWindowsZ3([string] $Path) { + $image = Get-WindowsImageDependencies $Path + return @($image.Dependencies | Where-Object { $_ -match '^(libz3|z3)\.dll$' }).Count -gt 0 +} diff --git a/tasks/scripts/windows-z3-tests.ps1 b/tasks/scripts/windows-z3-tests.ps1 new file mode 100644 index 0000000000..2e361f567f --- /dev/null +++ b/tasks/scripts/windows-z3-tests.ps1 @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +. (Join-Path $PSScriptRoot 'windows-z3.ps1') +$tokens = $null; $errors = $null +$ast = [Management.Automation.Language.Parser]::ParseFile((Join-Path $PSScriptRoot 'windows-msvc.ps1'), [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw ($errors | Out-String) } +$function = $ast.Find({ param($node) $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Get-Sha256' }, $true) +Invoke-Expression $function.Extent.Text + +$fixture = Join-Path ([IO.Path]::GetTempPath()) ('openshell-z3-package-' + [guid]::NewGuid().ToString('N')) +$Z3Version = '5.1.0' +$Z3SourceSha256 = 'e7ab3f8b682d5f55e813f951c7f9bb4fb9a48567ce6143d7adae23027e0e42a0' +$target = 'aarch64-pc-windows-msvc' +$count = 0 +function Expect-Rejection([scriptblock] $Action, [string] $Message) { + $rejected = $false + try { & $Action | Out-Null } catch { + if ($_.Exception.Message -notmatch $Message) { throw } + $rejected = $true + } + if (-not $rejected) { throw "Expected rejection matching: $Message" } +} +try { + $inputDir = Join-Path $fixture 'input' + New-Item -ItemType Directory -Path (Join-Path $inputDir 'lib'), (Join-Path $inputDir 'include') -Force | Out-Null + Set-Content -LiteralPath (Join-Path $inputDir 'lib/libz3.lib') -Value 'static-library-fixture' + Set-Content -LiteralPath (Join-Path $inputDir 'include/z3.h') -Value 'header-fixture' + Set-Content -LiteralPath (Join-Path $inputDir 'LICENSE.txt') -Value 'license-fixture' + $files = @(Get-ChildItem -LiteralPath $inputDir -File -Recurse | ForEach-Object { + @{ path = $_.FullName.Substring($inputDir.Length + 1).Replace('\', '/'); sha256 = (Get-Sha256 $_.FullName).ToLowerInvariant(); size = $_.Length } + }) + $build = @{ version = $Z3Version; sourceSha256 = $Z3SourceSha256; target = $target; runtime = 'MT'; lto = $false; configuration = 'Release'; msvcToolset = '14.44.35207'; files = $files } + $build | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $inputDir 'build.json') -Encoding UTF8 + Add-Type -AssemblyName System.IO.Compression.FileSystem + $testArchive = Join-Path $fixture 'package.zip' + [IO.Compression.ZipFile]::CreateFromDirectory($inputDir, $testArchive) + $testArchiveHash = (Get-Sha256 $testArchive).ToLowerInvariant() + $testManifestPath = Join-Path $fixture 'manifest.json' + $manifest = @{ artifactType = 'application/vnd.nvidia.openshell.z3.v1'; layers = @(@{ mediaType = 'application/zip'; digest = "sha256:$testArchiveHash" }) } + $manifest | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $testManifestPath -Encoding UTF8 + $pin = @{ manifestDigest = 'sha256:' + (Get-Sha256 $testManifestPath).ToLowerInvariant(); archiveSha256 = $testArchiveHash; buildManifestSha256 = (Get-Sha256 (Join-Path $inputDir 'build.json')).ToLowerInvariant() } + $Z3Pin = [pscustomobject]@{ repository = 'ghcr.io/nvidia/openshell/z3'; artifactType = $manifest.artifactType; msvcToolset = '14.44'; packages = [pscustomobject]@{ $target = [pscustomobject]$pin } } + $script:requests = @() + $script:unavailable = $false + $script:corruptManifest = $false + $script:corruptBlob = $false + function Invoke-RestMethod([string] $Uri, [int] $TimeoutSec) { + $script:requests += $Uri + if ($script:unavailable) { throw 'Registry unavailable' } + return @{ token = 'anonymous-test-token' } + } + function Invoke-WebRequest([switch] $UseBasicParsing, [string] $Uri, [hashtable] $Headers, [string] $OutFile, [int] $TimeoutSec) { + $script:requests += $Uri + if ($script:unavailable) { throw 'Registry unavailable' } + if ($Headers.Authorization -ne 'Bearer anonymous-test-token') { throw 'Wrong registry authorization' } + if ($Uri -match '/manifests/') { + Copy-Item -LiteralPath $testManifestPath -Destination $OutFile + if ($script:corruptManifest) { Add-Content -LiteralPath $OutFile -Value 'tampered' } + } else { + Copy-Item -LiteralPath $testArchive -Destination $OutFile + if ($script:corruptBlob) { Add-Content -LiteralPath $OutFile -Value 'tampered' } + } + } + $RepoRoot = Join-Path $fixture 'cold' + $resolved = Resolve-Z3Package $target + if (-not $resolved -or $script:requests.Count -ne 3) { throw 'Cold package download failed' } + if (Test-Path -LiteralPath (Join-Path $resolved 'package.zip')) { throw 'Cache retains a redundant compressed library copy' } + if ($script:requests[1] -notlike "*/manifests/$($pin.manifestDigest)" -or $script:requests[2] -notlike "*/blobs/sha256:$testArchiveHash") { throw 'Package download did not use immutable digest pins' } + $count++ + $script:unavailable = $true + if ((Resolve-Z3Package $target) -ne $resolved -or $script:requests.Count -ne 3) { throw 'Verified package cache was not reused offline' } + $count++ + Add-Content -LiteralPath (Join-Path $resolved 'lib/libz3.lib') -Value 'corruption' + Expect-Rejection { Resolve-Z3Package $target } 'file checksum mismatch' + $count++ + Copy-Item -LiteralPath (Join-Path $inputDir 'lib/libz3.lib') -Destination (Join-Path $resolved 'lib/libz3.lib') -Force + Add-Content -LiteralPath (Join-Path $resolved 'build.json') -Value 'corruption' + Expect-Rejection { Resolve-Z3Package $target } 'build manifest checksum mismatch' + $count++ + $RepoRoot = Join-Path $fixture 'offline' + if ($null -ne (Resolve-Z3Package $target)) { throw 'Unavailable package did not permit source fallback' } + $count++ + $script:unavailable = $false + $script:corruptManifest = $true + $RepoRoot = Join-Path $fixture 'bad-manifest' + Expect-Rejection { Resolve-Z3Package $target } 'manifest checksum mismatch' + $count++ + $script:corruptManifest = $false + $script:corruptBlob = $true + $RepoRoot = Join-Path $fixture 'bad-blob' + Expect-Rejection { Resolve-Z3Package $target } 'package checksum mismatch' + $count++ + $requestsBefore = $script:requests.Count + if ($null -ne (Resolve-Z3Package 'x86_64-pc-windows-msvc') -or $script:requests.Count -ne $requestsBefore) { throw 'Unpublished architecture did not permit source fallback before networking' } + $count++ + Copy-Item -LiteralPath (Join-Path $inputDir 'build.json') -Destination (Join-Path $resolved 'build.json') -Force + Expect-Rejection { Assert-Z3Package $resolved 'x86_64-pc-windows-msvc' ([pscustomobject]$pin) } 'does not match the pin' + $count++ + $Z3Version = '5.3.2' + Expect-Rejection { Assert-Z3Package $resolved $target ([pscustomobject]$pin) } 'does not match the pin' + $count++ + $Z3Version = '5.1.0' + $Z3Pin.msvcToolset = '14.52' + Expect-Rejection { Assert-Z3Package $resolved $target ([pscustomobject]$pin) } 'does not match the pin' + $Z3Pin.msvcToolset = '14.44' + $count++ + + # Exercise the command boundary: target changes must clear a prior library, + # and source feature/config options must reach Cargo rather than the harness. + $function = $ast.Find({ param($node) $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Invoke-VsCargo' }, $true) + Invoke-Expression $function.Extent.Text + $UserZ3Environment = @{ Z3_LIBRARY_PATH_OVERRIDE = $null; Z3_SYS_Z3_HEADER = $null; Z3_SRC_SOURCE_DIR = $null } + $StaticRuntime = $true + $WindowsBuildJobs = 2 + $TargetDir = Join-Path $fixture 'cargo-target' + $LogDir = $fixture + $WindowsCargoMutex = [Threading.Mutex]::new($false) + $script:forceSource = $false + $script:commands = @() + $script:libraries = @() + function Configure-Z3([string[]] $RustTargets) { + if ($env:Z3_LIBRARY_PATH_OVERRIDE) { throw 'A previous target leaked its Z3 library' } + $env:Z3_LIBRARY_PATH_OVERRIDE = "C:\z3\$($RustTargets[0])\lib" + $feature = if ($script:forceSource) { '--features openshell-prover/vendored-z3' } else { '' } + return [pscustomobject]@{ FromSource = $script:forceSource; WorkspaceFeatures = $feature; ServerFeatures = $feature; GatewayFeatures = $feature } + } + function rustup { $global:LASTEXITCODE = 0 } + function Resolve-VsDevCmd([string] $RustTarget) { return 'C:\vs\VsDevCmd.bat' } + function Get-VsTargetArch([string] $RustTarget) { if ($RustTarget -eq 'aarch64-pc-windows-msvc') { return 'arm64' }; return 'amd64' } + function Get-HostArch { return 'amd64' } + function Resolve-NinjaPath { return 'C:\tools\ninja.exe' } + function Add-PathEntry([string] $Directory) { } + function Reset-WindowsCMakeGeneratorCache([string] $RustTarget) { } + function cmd { + $script:commands += $args[-1] + $script:libraries += $env:Z3_LIBRARY_PATH_OVERRIDE + $global:LASTEXITCODE = 0 + } + try { + foreach ($rustTarget in @('x86_64-pc-windows-msvc', 'aarch64-pc-windows-msvc')) { + Invoke-VsCargo -RustTarget $rustTarget -CargoArgs "cargo build --target $rustTarget" -LogName "$rustTarget.log" + } + if ($script:libraries[0] -eq $script:libraries[1] -or $script:libraries[1] -notlike '*aarch64-pc-windows-msvc*') { throw 'Cargo targets did not resolve separate Z3 packages' } + $count++ + $script:forceSource = $true + Invoke-VsCargo -RustTarget $target -CargoArgs 'cargo clippy --workspace -- -D warnings' -LogName 'clippy.log' + if ($script:commands[-1] -notmatch '--features openshell-prover/vendored-z3 -- -D warnings') { throw 'Source feature reached the Clippy arguments instead of Cargo' } + $count++ + Invoke-VsCargo -RustTarget $target -CargoArgs 'cargo nextest run --workspace' -LogName 'nextest.log' + if ($script:commands[-1] -notmatch 'cargo nextest run --config ' -or $script:commands[-1] -notmatch 'windows-static.toml') { throw 'Nextest did not receive static/source Cargo configuration' } + $count++ + } finally { $WindowsCargoMutex.Dispose() } +} finally { Remove-Item -LiteralPath $fixture -Recurse -Force } +Write-Host "$count pinned Z3 OCI download/cache/integrity checks passed." diff --git a/tasks/scripts/windows-z3.ps1 b/tasks/scripts/windows-z3.ps1 new file mode 100644 index 0000000000..1ab1057c2d --- /dev/null +++ b/tasks/scripts/windows-z3.ps1 @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GHCR transports the ZIP as an OCI artifact. Native Windows consumers need +# only HTTPS and PowerShell; no container engine or registry CLI is required. +function Invoke-Z3PackageDownload([string] $Uri, [hashtable] $Headers, [string] $OutFile) { + try { + Invoke-WebRequest -UseBasicParsing -Uri $Uri -Headers $Headers -OutFile $OutFile -TimeoutSec 180 | Out-Null + return $true + } catch { + Write-Warning "Pinned Z3 package is unavailable at $Uri; using the pinned source fallback." + return $false + } +} + +function Assert-Z3Package([string] $Directory, [string] $RustTarget, $Pin) { + # Pin the manifest independently so CI caches only the extracted library, + # without storing a second compressed copy of the same large payload. + $manifestPath = Join-Path $Directory 'build.json' + if ((Get-Sha256 $manifestPath).ToLowerInvariant() -ne $Pin.buildManifestSha256) { throw 'Z3 package build manifest checksum mismatch.' } + $build = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + if ($build.version -ne $Z3Version -or $build.target -ne $RustTarget -or $build.runtime -ne 'MT' -or $build.lto -ne $false -or $build.configuration -ne 'Release' -or $build.sourceSha256 -ne $Z3SourceSha256 -or $build.msvcToolset -notlike "$($Z3Pin.msvcToolset).*") { + throw 'Z3 package version, architecture, source, compiler, or static runtime does not match the pin.' + } + foreach ($file in $build.files) { + if ($file.path -match '(^[/\\]|(^|[/\\])\.\.([/\\]|$)|:)') { throw 'Invalid Z3 package file path.' } + $path = Join-Path $Directory $file.path + if (-not (Test-Path -LiteralPath $path -PathType Leaf) -or (Get-Sha256 $path).ToLowerInvariant() -ne $file.sha256) { throw "Z3 package file checksum mismatch: $($file.path)" } + } + foreach ($required in @('lib/libz3.lib', 'include/z3.h', 'LICENSE.txt')) { + if (-not (Test-Path -LiteralPath (Join-Path $Directory $required) -PathType Leaf)) { throw "Incomplete Z3 package: $required" } + } +} + +function Resolve-Z3Package([string] $RustTarget) { + $property = $Z3Pin.packages.PSObject.Properties[$RustTarget] + if ($null -eq $property) { return $null } + $pin = $property.Value + if ($pin.manifestDigest -notmatch '^sha256:[0-9a-f]{64}$' -or $pin.archiveSha256 -notmatch '^[0-9a-f]{64}$' -or $pin.buildManifestSha256 -notmatch '^[0-9a-f]{64}$') { throw 'Invalid Z3 package digest pin.' } + $cacheRoot = Join-Path $RepoRoot '.cache/windows-z3' + $destination = Join-Path $cacheRoot "$RustTarget-$($pin.archiveSha256)" + if (Test-Path -LiteralPath $destination -PathType Container) { + Assert-Z3Package $destination $RustTarget $pin + Write-Host "==> Z3: cached static /MT $Z3Version package for $RustTarget" + return $destination + } + $repository = $Z3Pin.repository + if ($repository -notmatch '^ghcr\.io/(nvidia/openshell/[a-z0-9/-]+)$') { throw 'Invalid OpenShell Z3 registry repository.' } + $name = $Matches[1] + $scope = [uri]::EscapeDataString("repository:${name}:pull") + try { + # Public packages keep developer machines and external CI credential-free. + $token = (Invoke-RestMethod -Uri "https://ghcr.io/token?service=ghcr.io&scope=$scope" -TimeoutSec 30).token + } catch { + Write-Warning "Pinned static Z3 $Z3Version package is unavailable from GHCR; using the pinned source fallback." + return $null + } + $headers = @{ Authorization = "Bearer $token"; Accept = 'application/vnd.oci.image.manifest.v1+json' } + $staging = Join-Path $cacheRoot ('download-' + [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $staging -Force | Out-Null + try { + $manifestPath = Join-Path $staging 'manifest.json' + if (-not (Invoke-Z3PackageDownload "https://ghcr.io/v2/$name/manifests/$($pin.manifestDigest)" $headers $manifestPath)) { return $null } + if ('sha256:' + (Get-Sha256 $manifestPath).ToLowerInvariant() -ne $pin.manifestDigest) { throw 'Z3 OCI manifest checksum mismatch.' } + $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + if ($manifest.artifactType -ne $Z3Pin.artifactType -or @($manifest.layers).Count -ne 1) { throw 'Unexpected Z3 OCI artifact layout.' } + $layer = $manifest.layers[0] + if ($layer.mediaType -ne 'application/zip' -or $layer.digest -ne "sha256:$($pin.archiveSha256)") { throw 'Z3 OCI layer does not match the package pin.' } + $archive = Join-Path $staging 'package.zip' + if (-not (Invoke-Z3PackageDownload "https://ghcr.io/v2/$name/blobs/$($layer.digest)" $headers $archive)) { return $null } + if ((Get-Sha256 $archive).ToLowerInvariant() -ne $pin.archiveSha256) { throw 'Downloaded Z3 package checksum mismatch.' } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $unpack = Join-Path $staging 'unpack' + [IO.Compression.ZipFile]::ExtractToDirectory($archive, $unpack) + Assert-Z3Package $unpack $RustTarget $pin + # Another native task may have populated this same immutable cache. + $installed = $false + if (-not (Test-Path -LiteralPath $destination)) { + try { [IO.Directory]::Move($unpack, $destination); $installed = $true } catch { + if (-not (Test-Path -LiteralPath $destination)) { throw } + } + } + if (-not $installed) { Assert-Z3Package $destination $RustTarget $pin } + Write-Host "==> Z3: downloaded static /MT $Z3Version package for $RustTarget ($($pin.manifestDigest))" + return $destination + } finally { Remove-Item -LiteralPath $staging -Recurse -Force } +} diff --git a/tasks/windows-z3.json b/tasks/windows-z3.json new file mode 100644 index 0000000000..c5f7b7fb8c --- /dev/null +++ b/tasks/windows-z3.json @@ -0,0 +1,22 @@ +{ + "version": "5.1.0", + "sourceSha256": "e7ab3f8b682d5f55e813f951c7f9bb4fb9a48567ce6143d7adae23027e0e42a0", + "repository": "ghcr.io/nvidia/openshell/z3", + "artifactType": "application/vnd.nvidia.openshell.z3.v1", + "packageRevision": 1, + "msvcToolset": "14.44", + "packages": { + "x86_64-pc-windows-msvc": { + "tag": "5.1.0-x86_64-pc-windows-msvc-mt-v1", + "manifestDigest": "sha256:70e411597fb978cd712c61d3bd06e5425a8c01001c78fc6681614a89b8a15726", + "archiveSha256": "6d3fbe95d930d92e7832e95d82b9c5bbaf33eb6c419dc50853e33ec543674a5e", + "buildManifestSha256": "cbb0923d2c5f1bc6d92b669f78c8a73c43147da44e1926cc9d01083565c8828f" + }, + "aarch64-pc-windows-msvc": { + "tag": "5.1.0-aarch64-pc-windows-msvc-mt-v1", + "manifestDigest": "sha256:8dd68a756527fe9b94e987646a5f2c67c5703761560c24bb2ddd9e5ea7031387", + "archiveSha256": "8401565ff92a9b247bfa39688202eb2ff479ecaeef684a6848bd3bc5b5e63f6a", + "buildManifestSha256": "2989fa30c2e9379c6964adba23107016097cebd271c74fa3cce3cf726efeb419" + } + } +} diff --git a/tasks/windows.toml b/tasks/windows.toml index d991b1d5e9..8b265c5186 100644 --- a/tasks/windows.toml +++ b/tasks/windows.toml @@ -125,3 +125,12 @@ run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts description = "Validate MXC audit fixtures and real seven-event JSONL coverage (requires ETW privileges and native release binaries)" run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-mxc-audit native" +["windows:z3:package:x64"] +description = "Build the pinned static /MT Z3 x64 dependency package" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 package-z3 x86_64-pc-windows-msvc" + +["windows:z3:package:arm64"] +description = "Build the pinned static /MT Z3 ARM64 dependency package" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 package-z3 aarch64-pc-windows-msvc"