From d6e685c379b8c6f6de980a536f7f87ddbc932142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sat, 31 Oct 2026 14:03:41 -0400 Subject: [PATCH 1/8] Add native preindexed repository publishing in Rust and C# Provide reusable publishers, matching repository commands, immutable snapshots and signed static publication. Harden manifest merging, byte-level parity, state validation and rollback behavior after review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 28 + .github/workflows/release.yml | 13 +- .github/workflows/repository-build.yml | 53 + README.md | 4 + docs/repository-publishing.md | 226 + dotnet/Devolutions.Pinget.slnx | 2 + .../Devolutions.Pinget.Cli.csproj | 1 + dotnet/src/Devolutions.Pinget.Cli/Program.cs | 1 + .../PublisherCommands.cs | 204 + .../CoreTests.cs | 45 +- .../MsZipReaderTests.cs | 117 + .../Devolutions.Pinget.Core.csproj | 1 + .../Devolutions.Pinget.Core/NativeMsZip.cs | 95 + .../PreIndexedSource.cs | 9 +- .../Devolutions.Pinget.Publisher.Tests.csproj | 23 + .../PublisherPackagingTests.cs | 513 +++ .../PublisherWorkspaceTests.cs | 991 +++++ .../Devolutions.Pinget.Publisher.csproj | 27 + .../ManifestScanner.cs | 715 +++ .../MsZipCompressor.cs | 111 + .../PublisherModels.cs | 53 + .../PublisherPackaging.cs | 563 +++ .../PublisherWorkspace.cs | 586 +++ .../Sql/index-schema.sql | 20 + .../Sql/state-schema.sql | 3 + .../WinGetV2IndexWriter.cs | 166 + rust/Cargo.toml | 1 + rust/crates/pinget-cli/Cargo.toml | 1 + rust/crates/pinget-cli/src/main.rs | 230 +- rust/crates/pinget-core/src/lib.rs | 206 +- rust/crates/pinget-publisher/Cargo.toml | 25 + .../pinget-publisher/src/index-schema.sql | 53 + rust/crates/pinget-publisher/src/lib.rs | 3931 +++++++++++++++++ .../pinget-publisher/src/state-schema.sql | 10 + .../pinget-publisher/tests/conformance.rs | 1594 +++++++ rust/release-plz.toml | 6 +- scripts/Set-PingetVersion.ps1 | 17 +- tests/publisher/bootstrap-winget-sandbox.ps1 | 116 + tests/publisher/build_v2_fixture.py | 392 ++ .../publisher/compare_production_snapshots.py | 258 ++ tests/publisher/contract.md | 175 + tests/publisher/fixtures/AppxManifest.xml.in | 43 + .../Pinget.Test.Matrix_1.10.0.installer.yaml | 37 + ...inget.Test.Matrix_1.10.0.locale.en-US.yaml | 28 + ...inget.Test.Matrix_1.10.0.locale.fr-FR.yaml | 10 + .../Pinget.Test.Matrix_1.10.0.version.yaml | 5 + .../multi/Pinget.Test.Matrix_1.9.0.yaml | 17 + tests/publisher/fixtures/singleton.yaml.in | 19 + .../singleton/Pinget.Contract.Singleton.yaml | 19 + tests/publisher/index-schema.sql | 66 + .../publisher/run-production-conformance.ps1 | 285 ++ .../publisher/run-sandbox-publisher-gate.ps1 | 177 + tests/publisher/sandbox-conformance.ps1 | 719 +++ tests/publisher/serve-feed.ps1 | 136 + .../publisher/validate_production_snapshot.py | 202 + 55 files changed, 13322 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/repository-build.yml create mode 100644 docs/repository-publishing.md create mode 100644 dotnet/src/Devolutions.Pinget.Cli/PublisherCommands.cs create mode 100644 dotnet/src/Devolutions.Pinget.Core.Tests/MsZipReaderTests.cs create mode 100644 dotnet/src/Devolutions.Pinget.Core/NativeMsZip.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj create mode 100644 dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherWorkspaceTests.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/Devolutions.Pinget.Publisher.csproj create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/ManifestScanner.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/MsZipCompressor.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/PublisherModels.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/PublisherPackaging.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/PublisherWorkspace.cs create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/Sql/index-schema.sql create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/Sql/state-schema.sql create mode 100644 dotnet/src/Devolutions.Pinget.Publisher/WinGetV2IndexWriter.cs create mode 100644 rust/crates/pinget-publisher/Cargo.toml create mode 100644 rust/crates/pinget-publisher/src/index-schema.sql create mode 100644 rust/crates/pinget-publisher/src/lib.rs create mode 100644 rust/crates/pinget-publisher/src/state-schema.sql create mode 100644 rust/crates/pinget-publisher/tests/conformance.rs create mode 100644 tests/publisher/bootstrap-winget-sandbox.ps1 create mode 100644 tests/publisher/build_v2_fixture.py create mode 100644 tests/publisher/compare_production_snapshots.py create mode 100644 tests/publisher/contract.md create mode 100644 tests/publisher/fixtures/AppxManifest.xml.in create mode 100644 tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.installer.yaml create mode 100644 tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.en-US.yaml create mode 100644 tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml create mode 100644 tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.version.yaml create mode 100644 tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.9.0.yaml create mode 100644 tests/publisher/fixtures/singleton.yaml.in create mode 100644 tests/publisher/fixtures/singleton/Pinget.Contract.Singleton.yaml create mode 100644 tests/publisher/index-schema.sql create mode 100644 tests/publisher/run-production-conformance.ps1 create mode 100644 tests/publisher/run-sandbox-publisher-gate.ps1 create mode 100644 tests/publisher/sandbox-conformance.ps1 create mode 100644 tests/publisher/serve-feed.ps1 create mode 100644 tests/publisher/validate_production_snapshot.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82d673e..47a0e4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,12 @@ jobs: - name: Test pinget-cli run: cargo test -p pinget-cli --manifest-path rust/Cargo.toml + - name: Test pinget-publisher + run: cargo test -p pinget-publisher --manifest-path rust/Cargo.toml + + - name: Check publisher documentation + run: cargo doc -p pinget-publisher --manifest-path rust/Cargo.toml --no-deps + - name: Build pinget-cli run: cargo build -p pinget-cli --manifest-path rust/Cargo.toml @@ -107,6 +113,27 @@ jobs: - name: Run core tests run: dotnet test dotnet/src/Devolutions.Pinget.Core.Tests/Devolutions.Pinget.Core.Tests.csproj -c Release --no-build + - name: Run publisher tests + run: dotnet test dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj -c Release + + - name: Check Rust and .NET publisher interoperability + shell: pwsh + run: | + cargo build --manifest-path rust/Cargo.toml -p pinget-cli --release + if ($LASTEXITCODE -ne 0) { throw 'Rust publisher CLI build failed.' } + $sdkRoot = Join-Path ${env:ProgramFiles(x86)} 'Windows Kits\10\bin' + $makeAppx = Get-ChildItem -LiteralPath $sdkRoot -Directory | + Sort-Object Name -Descending | + ForEach-Object { Join-Path $_.FullName 'x64\makeappx.exe' } | + Where-Object { Test-Path -LiteralPath $_ } | + Select-Object -First 1 + if (-not $makeAppx) { throw 'Windows SDK MakeAppx was not found.' } + & .\tests\publisher\run-production-conformance.ps1 ` + -RustCli .\rust\target\release\pinget.exe ` + -DotnetAssembly .\dotnet\src\Devolutions.Pinget.Cli\bin\Release\net10.0\pinget.dll ` + -MakeAppxPath $makeAppx + if ($LASTEXITCODE -ne 0) { throw 'Publisher interoperability checks failed.' } + - name: Build native COM DLLs shell: pwsh env: @@ -124,6 +151,7 @@ jobs: run: | New-Item -Path artifacts/ci-nuget -ItemType Directory -Force | Out-Null dotnet pack dotnet/src/Devolutions.Pinget.Core/Devolutions.Pinget.Core.csproj -c Release --no-build -o artifacts/ci-nuget + dotnet pack dotnet/src/Devolutions.Pinget.Publisher/Devolutions.Pinget.Publisher.csproj -c Release --no-build -o artifacts/ci-nuget dotnet pack dotnet/src/Devolutions.Pinget.PowerShell.Engine/Devolutions.Pinget.PowerShell.Engine.csproj -c Release --no-build -o artifacts/ci-nuget - name: Validate PowerShell module packaging diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 602b3dc..6d028ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,18 +120,20 @@ jobs: if ([string]::IsNullOrWhiteSpace($Version)) { $coreVersion = Select-String -Path 'rust/crates/pinget-core/Cargo.toml' -Pattern '^version = "([^"]+)"$' | Select-Object -First 1 $cliVersion = Select-String -Path 'rust/crates/pinget-cli/Cargo.toml' -Pattern '^version = "([^"]+)"$' | Select-Object -First 1 + $publisherVersion = Select-String -Path 'rust/crates/pinget-publisher/Cargo.toml' -Pattern '^version = "([^"]+)"$' | Select-Object -First 1 $dotnetVersion = Select-String -Path 'dotnet/src/Devolutions.Pinget.Cli/Program.cs' -Pattern '^const string Version = "([^"]+)";$' | Select-Object -First 1 - if (($null -eq $coreVersion) -or ($null -eq $cliVersion) -or ($null -eq $dotnetVersion)) { + if (($null -eq $coreVersion) -or ($null -eq $cliVersion) -or ($null -eq $publisherVersion) -or ($null -eq $dotnetVersion)) { throw 'Unable to detect version from source files' } $coreVersionValue = $coreVersion.Matches[0].Groups[1].Value $cliVersionValue = $cliVersion.Matches[0].Groups[1].Value + $publisherVersionValue = $publisherVersion.Matches[0].Groups[1].Value $dotnetVersionValue = $dotnetVersion.Matches[0].Groups[1].Value - if (($coreVersionValue -ne $cliVersionValue) -or ($coreVersionValue -ne $dotnetVersionValue)) { - throw "Version mismatch detected: pinget-core=$coreVersionValue pinget-cli=$cliVersionValue dotnet=$dotnetVersionValue" + if (($coreVersionValue -ne $cliVersionValue) -or ($coreVersionValue -ne $publisherVersionValue) -or ($coreVersionValue -ne $dotnetVersionValue)) { + throw "Version mismatch detected: pinget-core=$coreVersionValue pinget-cli=$cliVersionValue pinget-publisher=$publisherVersionValue dotnet=$dotnetVersionValue" } $Version = $coreVersionValue @@ -374,6 +376,9 @@ jobs: - name: Run core tests run: dotnet test dotnet/src/Devolutions.Pinget.Core.Tests/Devolutions.Pinget.Core.Tests.csproj -c Release --no-build + - name: Run publisher tests + run: dotnet test dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj -c Release + - name: Build native COM DLLs shell: pwsh env: @@ -443,6 +448,7 @@ jobs: $BinaryRoots = @( 'dotnet/src/Devolutions.Pinget.Core/bin/Release', + 'dotnet/src/Devolutions.Pinget.Publisher/bin/Release', 'dotnet/src/Devolutions.Pinget.PowerShell.Engine/bin/Release', 'dotnet/src/Devolutions.Pinget.PowerShell.Cmdlets/bin/Release', 'rust/target/x86_64-pc-windows-msvc/release', @@ -477,6 +483,7 @@ jobs: New-Item -Path dist/nuget -ItemType Directory -Force | Out-Null $packArgs = @('-c', 'Release', '--no-build', '-p:ContinuousIntegrationBuild=true', "-p:Version=$env:PACKAGE_VERSION", '-p:IncludeSymbols=true', '-p:SymbolPackageFormat=snupkg', '-o', 'dist/nuget') dotnet pack dotnet/src/Devolutions.Pinget.Core/Devolutions.Pinget.Core.csproj @packArgs + dotnet pack dotnet/src/Devolutions.Pinget.Publisher/Devolutions.Pinget.Publisher.csproj @packArgs dotnet pack dotnet/src/Devolutions.Pinget.PowerShell.Engine/Devolutions.Pinget.PowerShell.Engine.csproj @packArgs - name: Package PowerShell module diff --git a/.github/workflows/repository-build.yml b/.github/workflows/repository-build.yml new file mode 100644 index 0000000..647ff75 --- /dev/null +++ b/.github/workflows/repository-build.yml @@ -0,0 +1,53 @@ +name: Repository source build + +on: + workflow_dispatch: + inputs: + manifests: + description: Repository-relative directory containing WinGet manifests + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: Build an unsigned source snapshot + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Rust + shell: pwsh + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Build publisher CLI + run: cargo build --manifest-path rust/Cargo.toml --package pinget-cli --release + + - name: Build and verify snapshot + shell: pwsh + env: + MANIFEST_INPUT: ${{ inputs.manifests }} + run: | + $ErrorActionPreference = 'Stop' + $inputRoot = [System.IO.Path]::GetFullPath((Join-Path $env:GITHUB_WORKSPACE $env:MANIFEST_INPUT)) + $checkoutPrefix = [System.IO.Path]::TrimEndingDirectorySeparator($env:GITHUB_WORKSPACE) + [System.IO.Path]::DirectorySeparatorChar + if (-not $inputRoot.StartsWith($checkoutPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + throw 'Manifest directory must be inside the checkout.' + } + $stateRoot = Join-Path $env:RUNNER_TEMP 'publisher-state' + & .\rust\target\release\pinget.exe repository build --input $inputRoot --state $stateRoot + if ($LASTEXITCODE -ne 0) { throw 'Source build failed.' } + & .\rust\target\release\pinget.exe repository verify --state $stateRoot + if ($LASTEXITCODE -ne 0) { throw 'Snapshot verification failed.' } + + - name: Upload source snapshot + uses: actions/upload-artifact@v7 + with: + name: unsigned-source-snapshot + path: ${{ runner.temp }}/publisher-state/builds + if-no-files-found: error diff --git a/README.md b/README.md index 03f336b..80f63d1 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ Structured manifest output is also supported: - `show --output json|yaml` - `search --manifests --output json|yaml` +For authoring a static Windows package source, see +[Publishing a static WinGet source](docs/repository-publishing.md). The +`repository` commands are separate from client-side `source` management. + The C# implementation also ships the **`Devolutions.Pinget.Client` PowerShell 7 module** that mirrors the upstream `Microsoft.WinGet.Client` cmdlet family with renamed `Pinget` nouns, backed by `Devolutions.Pinget.Core`. ## Pinget CLI diff --git a/docs/repository-publishing.md b/docs/repository-publishing.md new file mode 100644 index 0000000..adb73ea --- /dev/null +++ b/docs/repository-publishing.md @@ -0,0 +1,226 @@ +# Publishing a static WinGet source + +Pinget's repository publisher converts Windows WinGet YAML manifests into a v2 +`Microsoft.PreIndexed.Package` source. The Rust `pinget-publisher` crate and +the .NET `Devolutions.Pinget.Publisher` library expose the same workflow through +their respective `pinget repository` commands. + +For embedding, use the `Pinget.Publisher` namespace in .NET. The publisher +library is separate from the client `Devolutions.Pinget.Core` package. + +This is publishing tooling, not a server. Clients need an ordinary static HTTPS +host, not a REST service, Git checkout or access to the build-state database. +Installer binaries can remain at their immutable vendor or internal storage +URLs; the build does not download or execute them. + +## Requirements + +- Standard Windows WinGet manifests, either singleton or complete multifile + sets (version, installer, default locale and optional additional locales). +- Separate directories for inputs, publisher state and published files. + Do not put one inside another or use symlinks/reparse points in them. +- For publication on Windows, the Windows SDK's `makeappx.exe` and + `signtool.exe`. `repository doctor` reports tool availability; explicit tool + paths can be supplied when they are not discoverable. +- A code-signing certificate with its private key in a PFX, and a publisher + distinguished name matching that certificate. + +Building, verification and publication currently require Windows because +sidecars use the native Windows Compression API. `status` and `diff` can +inspect state on other platforms; `doctor` reports the missing capability. +Do not infer publishing support from the availability of the CLI on a platform. +Unsupported toolchains fail rather than silently produce unsigned output. + +## Build, inspect and publish + +Use the Rust or .NET CLI, not both at once against the same state directory. +The directories below are siblings, outside the input manifest tree: + +```powershell +pinget repository doctor +pinget repository build --input .\manifests --state .\publisher-state +pinget repository diff --input .\manifests --state .\publisher-state --output json +pinget repository status --state .\publisher-state --output json +pinget repository verify --state .\publisher-state +``` + +`--manifests` is an alias for `--input`. `--output` retains its existing meaning +of `text`, `json` or `yaml`; it is not a directory option. `diff` compares raw +input content with the last successful build. Publication is a separate state +transition. Inspection commands do not create missing state. + +Builds produce immutable snapshots, not a mutable directory that publication +rescans later. The build identifier is derived from the sorted input inventory +and publisher-format version. A failed build must not replace the previous +successful snapshot. The build-state database is private publisher bookkeeping: +never host it or substitute it for `Public/index.db`. + +Each package version produces one merged manifest and one version-data record. +The default locale remains the main manifest; additional locales are retained +under `Localization`, not exposed as duplicate package versions. + +The state and snapshots record `artifactFormatVersion=2`. Older publisher +state is rejected without automatic migration; use a new state directory +rather than editing its schema markers. Existing version-1 destination +ownership records can still be verified and updated through normal publication. + +State lives in `publisher.db`. Each snapshot is under `builds\`, +with `snapshot.json`, `Public\index.db` and hosted files under +`feed\manifests` and `feed\packages`. Use `--build ` with `publish` +or state verification to select a snapshot other than the latest. + +Prepare the PFX outside both the input and published directories. Supply the +password to the process through your CI secret mechanism, not a literal command +line or checked-in file: + +```powershell +# SOURCE_PFX_PASSWORD is already populated by the CI secret store. +pinget repository publish --state .\publisher-state --destination .\public ` + --name Pinget.InternalSource --publisher 'CN=Example Source Publisher' ` + --version 1.0.0.0 --pfx .\secrets\source.pfx ` + --password-env SOURCE_PFX_PASSWORD + +pinget repository verify --destination .\public +``` + +The source package version is an explicit **four-component MSIX version**, +separate from any application version. Keep the source identity and publisher +stable, and increase the source version on each changed publication. + +For local format development only, replace the signing options with +`--unsigned-development`. This is not a production trust configuration. The +publisher does not install certificates, configure WinGet source trust or +change the caller's sources. + +The PFX password is not stored in publisher state or normal diagnostics. +`signtool`'s PFX interface requires passing it to the signing child process; +privileged local process inspection can therefore observe it. Use a protected +build agent and restrict access to both that agent and the PFX. The password +environment-variable indirection is not a guarantee of process-level secret +isolation. + +## Host the result + +Copy the **entire published directory**, maintaining relative paths: + +| Published file | Purpose | +| --- | --- | +| `source2.msix` | Signed source package containing `Public/index.db` | +| `manifests/.yaml` | Merged manifest bytes referenced by version data | +| `packages///versionData.mszyml` | Compressed version metadata referenced by the index | + +A static website, reverse proxy, object-storage HTTPS endpoint or CDN can serve +these files. Enable `GET` and `HEAD`; retain `ETag`/`Last-Modified` metadata and +conditional requests. Ensure the host serves `.msix`, `.mszyml` and `.yaml` +rather than rejecting unknown extensions or rewriting them to a website page. +Use `application/octet-stream` when no more specific type is configured. +Range requests for large source packages are useful. + +Use a short cache lifetime or revalidation for `source2.msix`. Hash-addressed +manifests and sidecars may be cached for a long time. A CDN must not serve a +new root package before its referenced objects are readable through that CDN. + +The local publisher promotes new immutable metadata **before** replacing +`source2.msix`. A later cloud upload is a separate operation: upload new +manifests and sidecars first, verify availability, then replace the root MSIX +last. A recursive sync does not inherently preserve that ordering. + +Keep old hash-addressed metadata for clients holding cached indexes. There is +no automatic garbage collection. Do not overwrite an existing content path +with different bytes, delete a destination marker, or mix unrelated website +content into a publisher-owned destination. + +## Configure clients and trust + +Distribute the public certificate/chain through your normal endpoint-management +process before registering the source. Never distribute the PFX/private key. +A source-signing certificate and the HTTPS server certificate have different +jobs: MSIX signing authenticates source metadata; HTTPS authenticates transport. +Installer hashes and installer Authenticode checks remain separate. + +Use a certificate chain trusted by your endpoints. A self-signed development +certificate requires an explicit trust decision in an isolated test machine; +do not add a test root certificate to production machines as a shortcut. + +Register the **base URL**, not the build-state directory or a direct MSIX URL: + +```powershell +winget source add --name InternalApps --arg https://packages.example.com/apps ` + --type Microsoft.PreIndexed.Package +winget source update --name InternalApps +winget search --source InternalApps +winget show --id Example.App --exact --source InternalApps +``` + +The publishing commands never run these client commands automatically. +Hosting authentication and endpoint credentials are the host/client's concern: +this publisher does not add a secret store, token broker or authenticated +REST-source protocol. + +## CI and compatibility + +In CI, persist the publisher state between runs if you want meaningful diffs, +but keep it separate from deployment output. Run `build`, inspect `diff`, run +`verify`, then publish with the next source version. Upload to a staging +location before index-last promotion. Restrict signing credentials to trusted +branches and protected deployment jobs; do not expose them to pull-request code. + +The manual [source-build workflow](../.github/workflows/repository-build.yml) +is a minimal unsigned build example: give it a repository-relative manifest +directory, and it uploads a verified snapshot for inspection. It does not sign, +register a source or deploy a feed. Add signing and index-last deployment only +in a protected job appropriate for your hosting environment. + +The shared [format contract](../tests/publisher/contract.md) and independently +generated fixtures under `tests\publisher` record the compatibility baseline. +The initial full-index fixture was accepted by stock WinGet **1.29.380** in +Windows Sandbox for source add/update, search, show and hash-verified download. +That fixture result alone does not certify every publisher code path, older +client version, package manifest field, or signing toolchain. + +Current production conformance also covers matching Rust/.NET build identities, +canonical manifest and sidecar bytes, logical SQLite records, reciprocal +state reopening and unsigned destination verification. Before the +artifact-format-v2 review hardening, signed output from both publishers +passed stock WinGet 1.29.380 acceptance, including source +add/update, search/show, hash-verified download, version bumps, package +addition/removal and retention of metadata for cached indexes. Both Pinget +readers consumed both publishers' 500-version feeds and selected `1.0.499`, +exercising multi-block MSZIP sidecars end to end. These results establish the +tested client/toolchain baseline, not universal compatibility with older +clients or every manifest field. Current artifact-format-v2 outputs passed +the local exact-byte comparison and independent SQLite/MSZIP checks, but +have not received a fresh signed stock-client acceptance run. + +Failure tests cover packaging/signing errors, update promotion rollback and +destination locking. Initial publication failure after metadata copying, +followed by cleanup and retry, remains an end-to-end coverage gap. + +Stock-client acceptance and signature trust checks belong in a fresh Windows +Sandbox or an ephemeral Windows CI runner. They must not register test sources +or trust certificates in a developer's usual profile. + +This feature does not define a cross-platform manifest format, macOS package +installation, managed script bundles, assignment policy, a REST source, +background enforcement or new PowerShell cmdlets. + +Manifest ingestion accepts an explicit set of `ManifestVersion` values from +`1.0.0` through `1.10.0` and checks required fields, document roles and merged +structure. These checks are not a replacement for full upstream JSON-schema +validation of every manifest property. Search normalization reuses Pinget's +existing normalizers; compatibility evidence covers the tested fixtures, not +every possible name or publisher string. + +## Building and distributing the libraries + +The reusable Rust crate is `rust\crates\pinget-publisher`; the .NET project is +`dotnet\src\Devolutions.Pinget.Publisher`. Both existing CLI packages include +the repository commands. The .NET library targets .NET 8 and .NET 10. + +The Rust publisher depends on the matching core's normalization APIs. When +publishing crates, release the updated `pinget-core` before +`pinget-publisher`; building a packaged publisher against an older registry +core with the same version will not find those new APIs. The release-plz +workspace configuration publishes dependencies first. Use +`scripts\Set-PingetVersion.ps1` to keep package versions and dependency +requirements synchronized. diff --git a/dotnet/Devolutions.Pinget.slnx b/dotnet/Devolutions.Pinget.slnx index 92785ff..12e94ba 100644 --- a/dotnet/Devolutions.Pinget.slnx +++ b/dotnet/Devolutions.Pinget.slnx @@ -3,6 +3,8 @@ + + diff --git a/dotnet/src/Devolutions.Pinget.Cli/Devolutions.Pinget.Cli.csproj b/dotnet/src/Devolutions.Pinget.Cli/Devolutions.Pinget.Cli.csproj index 95b94dc..1dc361a 100644 --- a/dotnet/src/Devolutions.Pinget.Cli/Devolutions.Pinget.Cli.csproj +++ b/dotnet/src/Devolutions.Pinget.Cli/Devolutions.Pinget.Cli.csproj @@ -2,6 +2,7 @@ + diff --git a/dotnet/src/Devolutions.Pinget.Cli/Program.cs b/dotnet/src/Devolutions.Pinget.Cli/Program.cs index 9b4be03..38a19cf 100644 --- a/dotnet/src/Devolutions.Pinget.Cli/Program.cs +++ b/dotnet/src/Devolutions.Pinget.Cli/Program.cs @@ -1189,6 +1189,7 @@ rootCommand.AddCommand(uninstallCommand); rootCommand.AddCommand(repairCommand); rootCommand.AddCommand(importCommand); +rootCommand.AddCommand(PublisherCommands.Create(outputOption)); rootCommand.SetHandler((ctx) => { diff --git a/dotnet/src/Devolutions.Pinget.Cli/PublisherCommands.cs b/dotnet/src/Devolutions.Pinget.Cli/PublisherCommands.cs new file mode 100644 index 0000000..3b7ff46 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Cli/PublisherCommands.cs @@ -0,0 +1,204 @@ +using System.CommandLine; +using System.Text.Json; +using System.Text.Json.Nodes; +using Pinget.Publisher; + +namespace Devolutions.Pinget.Cli; + +internal static class PublisherCommands +{ + public static Command Create(Option outputOption) + { + var root = new Command("repository", "Build and publish WinGet v2 preindexed sources"); + + var stateBuild = new Option("--state", "Publisher working directory") { IsRequired = true }; + var inputBuild = InputOption(); + var build = new Command("build", "Build an immutable source snapshot"); + build.AddOption(stateBuild); + build.AddOption(inputBuild); + build.SetHandler(context => Run(context, outputOption, () => + { + var result = new PublisherWorkspace(context.ParseResult.GetValueForOption(stateBuild)!).Build( + new PublisherBuildOptions( + context.ParseResult.GetValueForOption(inputBuild)!, + context.ParseResult.GetValueForOption(stateBuild)!)); + return (Object( + ("buildId", JsonValue.Create(result.BuildId)), + ("snapshotDirectory", JsonValue.Create(result.SnapshotDirectory)), + ("manifestCount", JsonValue.Create(result.ManifestCount)), + ("packageCount", JsonValue.Create(result.PackageCount)), + ("changedFiles", JsonValue.Create(result.ChangedFiles)), + ("removedFiles", JsonValue.Create(result.RemovedFiles))), + $"Built {result.PackageCount} package(s), {result.ManifestCount} manifest(s). Build: {result.BuildId}"); + })); + + var stateDiff = new Option("--state", "Publisher working directory") { IsRequired = true }; + var inputDiff = InputOption(); + var diff = new Command("diff", "Compare manifests with the last successful build"); + diff.AddOption(stateDiff); + diff.AddOption(inputDiff); + diff.SetHandler(context => Run(context, outputOption, () => + { + var result = new PublisherWorkspace(context.ParseResult.GetValueForOption(stateDiff)!) + .Diff(context.ParseResult.GetValueForOption(inputDiff)!); + return (Object( + ("lastBuildId", JsonValue.Create(result.LastBuildId)), + ("published", JsonValue.Create(result.Published)), + ("added", Array(result.Added)), + ("changed", Array(result.Changed)), + ("removed", Array(result.Removed))), + $"Added {result.Added.Count}, changed {result.Changed.Count}, removed {result.Removed.Count}; last build published: {result.Published}"); + })); + + var stateStatus = new Option("--state", "Publisher working directory") { IsRequired = true }; + var status = new Command("status", "Show publisher state without rescanning manifests"); + status.AddOption(stateStatus); + status.SetHandler(context => Run(context, outputOption, () => + { + var result = new PublisherWorkspace(context.ParseResult.GetValueForOption(stateStatus)!).Status(); + return (Object( + ("lastBuildId", JsonValue.Create(result.LastBuildId)), + ("lastPublishedBuildId", JsonValue.Create(result.LastPublishedBuildId)), + ("buildCount", JsonValue.Create(result.BuildCount)), + ("lastBuildTimestamp", JsonValue.Create(result.LastBuildTimestamp))), + result.LastBuildId is null ? "No publisher builds have been recorded." : $"Builds: {result.BuildCount}; latest: {result.LastBuildId}; published: {result.LastPublishedBuildId ?? "never"}"); + })); + + var stateVerify = new Option("--state", "Publisher working directory"); + var buildVerify = new Option("--build", "Build snapshot ID"); + var destinationVerify = new Option("--destination", "Published static source directory"); + var signToolVerify = new Option("--signtool", "Path to signtool.exe"); + var verify = new Command("verify", "Verify a build snapshot or published directory"); + verify.AddOption(stateVerify); + verify.AddOption(buildVerify); + verify.AddOption(destinationVerify); + verify.AddOption(signToolVerify); + verify.SetHandler(context => Run(context, outputOption, () => + { + var state = context.ParseResult.GetValueForOption(stateVerify); + var destination = context.ParseResult.GetValueForOption(destinationVerify); + if ((state is null) == (destination is null)) + throw new PublisherException("Provide exactly one of --state or --destination."); + if (destination is not null && context.ParseResult.GetValueForOption(buildVerify) is not null) + throw new PublisherException("--build cannot be combined with --destination."); + var result = destination is not null + ? PublisherPackaging.VerifyDestination(destination, context.ParseResult.GetValueForOption(signToolVerify)) + : new PublisherWorkspace(state!).Verify(context.ParseResult.GetValueForOption(buildVerify)); + return (Object( + ("valid", JsonValue.Create(result.Valid)), + ("subject", JsonValue.Create(result.Subject)), + ("artifactCount", JsonValue.Create(result.ArtifactCount)), + ("buildId", JsonValue.Create(result.BuildId))), + $"Verified {result.Subject}; artifacts: {result.ArtifactCount}."); + })); + + var makeAppxDoctor = new Option("--makeappx", "Path to makeappx.exe"); + var signToolDoctor = new Option("--signtool", "Path to signtool.exe"); + var doctor = new Command("doctor", "Check local publisher capabilities and packaging tools"); + doctor.AddOption(makeAppxDoctor); + doctor.AddOption(signToolDoctor); + doctor.SetHandler(context => Run(context, outputOption, () => + { + var result = PublisherPackaging.Doctor( + context.ParseResult.GetValueForOption(makeAppxDoctor), + context.ParseResult.GetValueForOption(signToolDoctor)); + return (Object( + ("operatingSystem", JsonValue.Create(result.OperatingSystem)), + ("makeAppxPath", JsonValue.Create(result.MakeAppxPath)), + ("signToolPath", JsonValue.Create(result.SignToolPath)), + ("nativeMsZipAvailable", JsonValue.Create(result.NativeMsZipAvailable))), + $"MSZIP: {(result.NativeMsZipAvailable ? "available" : "unavailable")}; makeappx: {result.MakeAppxPath ?? "not found"}; signtool: {result.SignToolPath ?? "not found"}."); + })); + + var statePublish = new Option("--state", "Publisher working directory") { IsRequired = true }; + var destinationPublish = new Option("--destination", "Static source output directory") { IsRequired = true }; + var namePublish = new Option("--name", "MSIX package identity name") { IsRequired = true }; + var publisherPublish = new Option("--publisher", "MSIX publisher distinguished name") { IsRequired = true }; + var versionPublish = new Option("--version", "Four-part MSIX package version") { IsRequired = true }; + var pfxPublish = new Option("--pfx", "PFX file used to sign the source MSIX"); + var passwordEnvPublish = new Option("--password-env", "Environment variable containing the PFX password"); + var unsignedPublish = new Option("--unsigned-development", "Allow explicitly unsigned development publishing"); + var makeAppxPublish = new Option("--makeappx", "Path to makeappx.exe"); + var signToolPublish = new Option("--signtool", "Path to signtool.exe"); + var timestampPublish = new Option("--timestamp-url", "HTTPS RFC 3161 timestamp service URL"); + var buildPublish = new Option("--build", "Build snapshot ID (defaults to latest)"); + var publish = new Command("publish", "Package and publish a source2.msix and immutable artifacts"); + foreach (var option in new Option[] + { + statePublish, destinationPublish, namePublish, publisherPublish, versionPublish, + pfxPublish, passwordEnvPublish, unsignedPublish, makeAppxPublish, signToolPublish, buildPublish, timestampPublish, + }) + publish.AddOption(option); + publish.SetHandler(context => Run(context, outputOption, () => + { + var result = PublisherPackaging.Publish(new PublisherPublishOptions( + context.ParseResult.GetValueForOption(statePublish)!, + context.ParseResult.GetValueForOption(destinationPublish)!, + context.ParseResult.GetValueForOption(namePublish)!, + context.ParseResult.GetValueForOption(publisherPublish)!, + context.ParseResult.GetValueForOption(versionPublish)!, + context.ParseResult.GetValueForOption(pfxPublish), + context.ParseResult.GetValueForOption(passwordEnvPublish), + context.ParseResult.GetValueForOption(unsignedPublish), + context.ParseResult.GetValueForOption(makeAppxPublish), + context.ParseResult.GetValueForOption(signToolPublish), + context.ParseResult.GetValueForOption(buildPublish), + context.ParseResult.GetValueForOption(timestampPublish))); + return (Object( + ("valid", JsonValue.Create(result.Valid)), + ("subject", JsonValue.Create(result.Subject)), + ("artifactCount", JsonValue.Create(result.ArtifactCount)), + ("buildId", JsonValue.Create(result.BuildId))), + $"Published and verified source for build {result.BuildId}; artifacts: {result.ArtifactCount}."); + })); + + root.AddCommand(build); + root.AddCommand(diff); + root.AddCommand(status); + root.AddCommand(verify); + root.AddCommand(doctor); + root.AddCommand(publish); + return root; + } + + private static Option InputOption() + { + var option = new Option("--input", "Root directory containing WinGet YAML manifests") { IsRequired = true }; + option.AddAlias("--manifests"); + return option; + } + + private static void Run(System.CommandLine.Invocation.InvocationContext context, Option outputOption, Func<(JsonObject Result, string Text)> action) + { + try + { + var output = context.ParseResult.GetValueForOption(outputOption); + if (output is not null && output.ToLowerInvariant() is not ("text" or "json" or "yaml")) + throw new PublisherException("--output must be text, json, or yaml."); + var (result, text) = action(); + if (string.Equals(output, "json", StringComparison.OrdinalIgnoreCase)) + Console.WriteLine(result.ToJsonString(new JsonSerializerOptions { WriteIndented = true })); + else if (string.Equals(output, "yaml", StringComparison.OrdinalIgnoreCase)) + Console.WriteLine(Devolutions.Pinget.Core.YamlEmitter.EmitDocument(result)); + else + Console.WriteLine(text); + } + catch (Exception ex) + { + var diagnostics = Environment.GetEnvironmentVariable("PINGET_PUBLISHER_DIAGNOSTICS"); + Console.Error.WriteLine(diagnostics == "1" ? ex : $"error: {ex.Message}"); + context.ExitCode = 1; + } + } + + private static JsonObject Object(params (string Name, JsonNode? Value)[] values) + { + var result = new JsonObject(); + foreach (var (name, value) in values) + result[name] = value; + return result; + } + + private static JsonArray Array(IEnumerable values) => + new(values.Select(value => (JsonNode?)JsonValue.Create(value)).ToArray()); +} diff --git a/dotnet/src/Devolutions.Pinget.Core.Tests/CoreTests.cs b/dotnet/src/Devolutions.Pinget.Core.Tests/CoreTests.cs index 261c998..5d023e2 100644 --- a/dotnet/src/Devolutions.Pinget.Core.Tests/CoreTests.cs +++ b/dotnet/src/Devolutions.Pinget.Core.Tests/CoreTests.cs @@ -4294,25 +4294,34 @@ private static JsonDocument RunCliShowJson(string appRoot) { var root = FindRepositoryRoot(); var cliProject = Path.Combine(root, "dotnet", "src", "Devolutions.Pinget.Cli", "Devolutions.Pinget.Cli.csproj"); - var cliDll = Path.Combine(root, "dotnet", "src", "Devolutions.Pinget.Cli", "bin", "Release", "net10.0", "pinget.dll"); - - var build = RunProcess("dotnet", ["build", cliProject, "-c", "Release", "-v:q"], appRoot); - Assert.Equal(0, build.ExitCode); - Assert.True(File.Exists(cliDll)); + var artifacts = Path.Combine(root, "artifacts", "cli-embedding-tests", Guid.NewGuid().ToString("N")); + var cliDll = Path.Combine(artifacts, "bin", "Devolutions.Pinget.Cli", "release", "pinget.dll"); + try + { + var build = RunProcess("dotnet", + ["build", cliProject, "-c", "Release", "--artifacts-path", artifacts, "-v:q"], appRoot); + Assert.True(build.ExitCode == 0, $"CLI build failed:{Environment.NewLine}{build.Stdout}{Environment.NewLine}{build.Stderr}"); + Assert.True(File.Exists(cliDll)); - var run = RunProcess("dotnet", - [ - cliDll, - "show", - "--id", TesslPackageId, - "--exact", - "--source", "test", - "--architecture", "x64", - "--output", "json", - ], appRoot); - - Assert.Equal(0, run.ExitCode); - return JsonDocument.Parse(run.Stdout); + var run = RunProcess("dotnet", + [ + cliDll, + "show", + "--id", TesslPackageId, + "--exact", + "--source", "test", + "--architecture", "x64", + "--output", "json", + ], appRoot); + + Assert.Equal(0, run.ExitCode); + return JsonDocument.Parse(run.Stdout); + } + finally + { + if (Directory.Exists(artifacts)) + Directory.Delete(artifacts, recursive: true); + } } private static (int ExitCode, string Stdout, string Stderr) RunProcess(string fileName, IReadOnlyList arguments, string appRoot) diff --git a/dotnet/src/Devolutions.Pinget.Core.Tests/MsZipReaderTests.cs b/dotnet/src/Devolutions.Pinget.Core.Tests/MsZipReaderTests.cs new file mode 100644 index 0000000..d92f780 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Core.Tests/MsZipReaderTests.cs @@ -0,0 +1,117 @@ +using System.Buffers.Binary; +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Text; +using Devolutions.Pinget.Core; +using Xunit; +using Xunit.Sdk; + +namespace Devolutions.Pinget.Core.Tests; + +public class MsZipReaderTests +{ + [SkippableFact] + public void DecompressMszyml_NativeMultiBlockPayload_ReturnsAllVersionRecords() + { + Skip.IfNot(OperatingSystem.IsWindows(), "The fixture uses the Windows Compression API to generate native MSZIP framing."); + + const int versionCount = 500; + var payload = new StringBuilder("sV: \"1.0\"\nvD:\n"); + for (var i = 0; i < versionCount; i++) + { + payload.Append(" - v: 1.0.").Append(i).Append('\n'); + payload.Append(" rP: manifests/Pinget.MultiBlock/1.0.").Append(i).Append(".yaml\n"); + payload.Append(" s256H: ").Append('a', 64).Append('\n'); + } + + var payloadBytes = Encoding.UTF8.GetBytes(payload.ToString()); + Assert.True(payloadBytes.Length > 32768); + + var nativeMsZip = CompressWithWindowsApi(payloadBytes); + Assert.True(CountBlocks(nativeMsZip) > 1); + Assert.Equal(payloadBytes, NativeMsZip.DecompressManaged(nativeMsZip)); + + var decompressed = PreIndexedSource.DecompressMszyml(nativeMsZip); + Assert.Equal(payload.ToString(), decompressed); + + var parsed = PreIndexedSource.ParseV2VersionDataDocument(decompressed); + Assert.NotNull(parsed); + Assert.Equal(versionCount, parsed.Versions.Count); + Assert.Equal("1.0.499", parsed.Versions[^1].Version); + Assert.Equal("manifests/Pinget.MultiBlock/1.0.499.yaml", parsed.Versions[^1].ManifestRelativePath); + } + + [SkippableFact] + public void DecompressMszyml_TruncatedNativeMultiBlockPayload_RejectsCorruptFraming() + { + Skip.IfNot(OperatingSystem.IsWindows(), "The fixture uses the Windows Compression API to generate native MSZIP framing."); + + var payloadBytes = Encoding.UTF8.GetBytes("sV: \"1.0\"\nvD:\n" + string.Concat( + Enumerable.Range(0, 500).Select(i => + $" - v: 1.0.{i}\n rP: manifests/Pinget.MultiBlock/1.0.{i}.yaml\n s256H: {new string('b', 64)}\n"))); + var nativeMsZip = CompressWithWindowsApi(payloadBytes); + var truncated = nativeMsZip[..^1]; + + Assert.Throws(() => NativeMsZip.DecompressManaged(truncated)); + Assert.Throws(() => PreIndexedSource.DecompressMszyml(truncated)); + } + + private static int CountBlocks(byte[] data) + { + Assert.True(data.AsSpan().StartsWith(new byte[] { 0x0a, 0x51, 0xe5, 0xc0 })); + var position = 24; + var count = 0; + while (position < data.Length) + { + Assert.True(data.Length - position >= 6); + var compressedLength = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(position, 4)); + position += 4; + Assert.True(compressedLength >= 3); + Assert.Equal((byte)'C', data[position]); + Assert.Equal((byte)'K', data[position + 1]); + Assert.True(compressedLength <= data.Length - position); + position += (int)compressedLength; + count++; + } + + return count; + } + + private static byte[] CompressWithWindowsApi(byte[] input) + { + if (!CreateCompressor(2, IntPtr.Zero, out var compressor)) + throw new Win32Exception(Marshal.GetLastWin32Error()); + + try + { + _ = Compress(compressor, input, (nuint)input.Length, null, 0, out var required); + if (Marshal.GetLastWin32Error() != 122) + throw new Win32Exception(Marshal.GetLastWin32Error(), "MSZIP size query did not report insufficient buffer."); + + var output = new byte[checked((int)required)]; + if (!Compress(compressor, input, (nuint)input.Length, output, (nuint)output.Length, out var written)) + throw new Win32Exception(Marshal.GetLastWin32Error()); + + Array.Resize(ref output, checked((int)written)); + return output; + } + finally + { + _ = CloseCompressor(compressor); + } + } + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CreateCompressor(uint algorithm, IntPtr options, out IntPtr compressor); + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Compress( + IntPtr compressor, byte[] input, nuint inputSize, + byte[]? output, nuint outputSize, out nuint outputWritten); + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CloseCompressor(IntPtr compressor); +} diff --git a/dotnet/src/Devolutions.Pinget.Core/Devolutions.Pinget.Core.csproj b/dotnet/src/Devolutions.Pinget.Core/Devolutions.Pinget.Core.csproj index 62a8689..b076668 100644 --- a/dotnet/src/Devolutions.Pinget.Core/Devolutions.Pinget.Core.csproj +++ b/dotnet/src/Devolutions.Pinget.Core/Devolutions.Pinget.Core.csproj @@ -15,6 +15,7 @@ + diff --git a/dotnet/src/Devolutions.Pinget.Core/NativeMsZip.cs b/dotnet/src/Devolutions.Pinget.Core/NativeMsZip.cs new file mode 100644 index 0000000..39eca19 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Core/NativeMsZip.cs @@ -0,0 +1,95 @@ +using System.Buffers.Binary; +using System.IO.Compression; + +namespace Devolutions.Pinget.Core; + +internal static class NativeMsZip +{ + internal static byte[] DecompressManaged(byte[] input) + { + if (input.Length < 24) + throw new InvalidDataException("Truncated MSZIP framing header."); + var totalSize = BinaryPrimitives.ReadUInt64LittleEndian(input.AsSpan(8, 8)); + var blockSize = BinaryPrimitives.ReadUInt64LittleEndian(input.AsSpan(16, 8)); + if (totalSize > 64 * 1024 * 1024 || blockSize is 0 or > 32768) + throw new InvalidDataException("Invalid MSZIP size or block size (maximum output is 64 MiB)."); + var output = new byte[checked((int)totalSize)]; + var position = 24; + var written = 0; + while (position < input.Length) + { + if (input.Length - position < 4) + throw new InvalidDataException("Truncated MSZIP block length."); + var compressedLength = BinaryPrimitives.ReadUInt32LittleEndian(input.AsSpan(position, 4)); + position += 4; + if (compressedLength < 3 || compressedLength > input.Length - position + || input[position] != (byte)'C' || input[position + 1] != (byte)'K') + throw new InvalidDataException("Invalid or truncated MSZIP block."); + + var historyLength = Math.Min(written, 32768); + // A nonfinal stored block supplies the dictionary to the raw inflater. + var deflateBytes = new byte[checked(5 + historyLength + (int)compressedLength - 2)]; + BinaryPrimitives.WriteUInt16LittleEndian(deflateBytes.AsSpan(1), (ushort)historyLength); + BinaryPrimitives.WriteUInt16LittleEndian(deflateBytes.AsSpan(3), (ushort)~historyLength); + output.AsSpan(written - historyLength, historyLength).CopyTo(deflateBytes.AsSpan(5)); + input.AsSpan(position + 2, (int)compressedLength - 2).CopyTo(deflateBytes.AsSpan(5 + historyLength)); + using var compressed = new CompleteDeflateInput(deflateBytes); + using var inflater = new DeflateStream(compressed, CompressionMode.Decompress); + var block = new byte[historyLength + (int)blockSize]; + var read = 0; + while (read < block.Length) + { + var count = inflater.Read(block.AsSpan(read)); + if (count == 0) + break; + read += count; + } + if (inflater.ReadByte() != -1 || read <= historyLength || read - historyLength > output.Length - written) + throw new InvalidDataException("MSZIP block exceeds its declared output size."); + var produced = read - historyLength; + block.AsSpan(historyLength, produced).CopyTo(output.AsSpan(written)); + written += produced; + position += (int)compressedLength; + } + if (written != output.Length) + throw new InvalidDataException("MSZIP output does not match its declared size."); + return output; + } + + private sealed class CompleteDeflateInput(byte[] bytes) : Stream + { + private readonly MemoryStream input = new(bytes, false); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => input.Length; + public override long Position + { + get => input.Position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) => + Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + if (Position == Length && buffer.Length > 0) + throw new InvalidDataException("Truncated MSZIP deflate stream."); + return input.Read(buffer); + } + + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + input.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/dotnet/src/Devolutions.Pinget.Core/PreIndexedSource.cs b/dotnet/src/Devolutions.Pinget.Core/PreIndexedSource.cs index c4e4e8e..956f47f 100644 --- a/dotnet/src/Devolutions.Pinget.Core/PreIndexedSource.cs +++ b/dotnet/src/Devolutions.Pinget.Core/PreIndexedSource.cs @@ -614,7 +614,14 @@ private static bool HashMatches(string? expected, byte[] data) internal static string DecompressMszyml(byte[] bytes) { - // .mszyml files start with "CK" header then deflate-compressed data + // Windows framing carries block boundaries and MSZIP dictionary history. + if (bytes.AsSpan().StartsWith(new byte[] { 0x0a, 0x51, 0xe5, 0xc0 })) + { + var decompressed = NativeMsZip.DecompressManaged(bytes); + return System.Text.Encoding.UTF8.GetString(decompressed); + } + + // Preserve support for legacy single-block CK/raw-deflate sources. byte[] payload; if (bytes.Length >= 2 && bytes[0] == (byte)'C' && bytes[1] == (byte)'K') payload = bytes[2..]; diff --git a/dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj b/dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj new file mode 100644 index 0000000..9364c7b --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj @@ -0,0 +1,23 @@ + + + net10.0 + enable + enable + false + Devolutions.Pinget.Publisher.Tests + Devolutions.Pinget.Publisher.Tests + + + + + + + + + + + + + diff --git a/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs b/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs new file mode 100644 index 0000000..0edc0ec --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs @@ -0,0 +1,513 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Xml.Linq; +using Pinget.Publisher; +using Xunit; + +namespace Devolutions.Pinget.Publisher.Tests; + +public sealed class PublisherPackagingTests +{ + private const string PackageName = "Pinget.Publisher.Conformance"; + private const string Publisher = "CN=Pinget Publisher Conformance Test"; + + [Theory] + [InlineData("bad-name", Publisher, "1.0.0.0", "Package identity")] + [InlineData(PackageName, Publisher, "1.2.3", "four numeric components")] + [InlineData(PackageName, Publisher, "1.2.3.65536", "four numeric components")] + public void Publish_InvalidIdentityOrPackageVersion_IsRejected( + string packageName, + string publisher, + string version, + string diagnostic) + { + using var temp = new TempDirectory(); + var state = Path.Combine(temp.Path, "state"); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + _ = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var options = new PublisherPublishOptions( + state, + Path.Combine(temp.Path, "destination"), + packageName, + publisher, + version, + UnsignedDevelopment: true); + + var error = Assert.Throws(() => PublisherPackaging.Publish(options)); + + Assert.Contains(diagnostic, error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(Directory.Exists(options.DestinationDirectory)); + } + + [Fact] + public void Publish_LineFeedInPublisher_IsRejected() + { + using var temp = new TempDirectory(); + var state = Path.Combine(temp.Path, "state"); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + _ = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var options = new PublisherPublishOptions( + state, + Path.Combine(temp.Path, "destination"), + PackageName, + "CN=Invalid\nPublisher", + "1.0.0.0", + UnsignedDevelopment: true); + + var error = Assert.Throws(() => PublisherPackaging.Publish(options)); + + Assert.Contains("Publisher", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(Directory.Exists(options.DestinationDirectory)); + } + + [Fact] + public void Publish_RequiresExplicitUnsignedDevelopmentOrSigningConfiguration() + { + using var temp = new TempDirectory(); + var state = Path.Combine(temp.Path, "state"); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + _ = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var options = new PublisherPublishOptions( + state, + Path.Combine(temp.Path, "destination"), + PackageName, + Publisher, + "1.0.0.0"); + + var error = Assert.Throws(() => PublisherPackaging.Publish(options)); + + Assert.Contains("Choose exactly one", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(Directory.Exists(options.DestinationDirectory)); + } + + [Fact] + public void Publish_UnsignedDevelopment_EnforcesMonotonicVersionAndRetainsOldArtifacts() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var workspace = new PublisherWorkspace(state); + var firstBuild = workspace.Build(new PublisherBuildOptions(input, state)); + var options = UnsignedOptions(state, destination, "1.0.0.0", firstBuild.BuildId); + + var first = PublisherPackaging.Publish(options); + var repeated = PublisherPackaging.Publish(options); + Assert.True(first.Valid); + Assert.True(repeated.Valid); + Assert.Equal(first.BuildId, repeated.BuildId); + using (var marker = JsonDocument.Parse( + File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json")))) + { + Assert.Equal(1, marker.RootElement.GetProperty("schemaVersion").GetInt32()); + Assert.False(marker.RootElement.TryGetProperty("artifactFormatVersion", out _)); + } + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + AssertMsixContents( + Path.Combine(destination, "source2.msix"), + Path.Combine(firstBuild.SnapshotDirectory, "Public", "index.db"), + "1.0.0.0"); + + var oldRootIndex = File.ReadAllBytes(Path.Combine(destination, "source2.msix")); + var firstSnapshot = JsonDocument.Parse( + File.ReadAllBytes(Path.Combine(firstBuild.SnapshotDirectory, "snapshot.json"))); + var oldArtifacts = firstSnapshot.RootElement.GetProperty("artifacts") + .EnumerateObject() + .Select(property => property.Name) + .ToArray(); + + File.AppendAllText( + Path.Combine(input, "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"), + "# source content update\n", + new UTF8Encoding(false)); + var secondBuild = workspace.Build(new PublisherBuildOptions(input, state)); + Assert.NotEqual(firstBuild.BuildId, secondBuild.BuildId); + + var lowerVersion = UnsignedOptions(state, destination, "0.9.9.9", secondBuild.BuildId); + var downgrade = Assert.Throws(() => PublisherPackaging.Publish(lowerVersion)); + Assert.Contains("greater", downgrade.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(oldRootIndex, File.ReadAllBytes(Path.Combine(destination, "source2.msix"))); + + var nextVersion = UnsignedOptions(state, destination, "1.0.0.1", secondBuild.BuildId); + var updated = PublisherPackaging.Publish(nextVersion); + Assert.True(updated.Valid); + Assert.Equal(secondBuild.BuildId, updated.BuildId); + Assert.NotEqual( + Convert.ToHexString(SHA256.HashData(oldRootIndex)), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(Path.Combine(destination, "source2.msix"))))); + foreach (var relative in oldArtifacts) + Assert.True(File.Exists(Path.Combine(destination, relative.Replace('/', Path.DirectorySeparatorChar)))); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + } + + [Fact] + public void Publish_UnownedDestinationWithUnrelatedFile_IsRejectedWithoutMutation() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var build = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + Directory.CreateDirectory(destination); + var unrelatedPath = Path.Combine(destination, "keep-me.txt"); + var unrelatedBytes = Encoding.UTF8.GetBytes("unrelated destination content"); + File.WriteAllBytes(unrelatedPath, unrelatedBytes); + + var error = Assert.Throws(() => PublisherPackaging.Publish( + UnsignedOptions(state, destination, "1.0.0.0", build.BuildId))); + + Assert.Contains("non-empty unowned destination", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(unrelatedBytes, File.ReadAllBytes(unrelatedPath)); + Assert.False(File.Exists(Path.Combine(destination, ".pinget-publisher.json"))); + Assert.False(File.Exists(Path.Combine(destination, "source2.msix"))); + Assert.Null(new PublisherWorkspace(state).Status().LastPublishedBuildId); + } + + [Fact] + public void Publish_DifferentIdentityOnOwnedDestination_IsRejectedWithoutMutation() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + _ = PublisherPackaging.Publish(UnsignedOptions(state, destination, "1.0.0.0", build.BuildId)); + var oldIndex = File.ReadAllBytes(Path.Combine(destination, "source2.msix")); + var oldMarker = File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json")); + var changedIdentity = new PublisherPublishOptions( + state, + destination, + "Pinget.Publisher.Other", + Publisher, + "1.0.0.0", + UnsignedDevelopment: true, + BuildId: build.BuildId); + + var error = Assert.Throws(() => PublisherPackaging.Publish(changedIdentity)); + + Assert.Contains("different source package identity or publisher", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(oldIndex, File.ReadAllBytes(Path.Combine(destination, "source2.msix"))); + Assert.Equal(oldMarker, File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json"))); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + } + + [Fact] + public void Publish_UntrackedFileInOwnedDestination_IsRejectedAndPreserved() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var workspace = new PublisherWorkspace(state); + var firstBuild = workspace.Build(new PublisherBuildOptions(input, state)); + _ = PublisherPackaging.Publish(UnsignedOptions(state, destination, "1.0.0.0", firstBuild.BuildId)); + var oldIndex = File.ReadAllBytes(Path.Combine(destination, "source2.msix")); + var oldMarker = File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json")); + var unrelatedPath = Path.Combine(destination, "keep-me.txt"); + var unrelatedBytes = Encoding.UTF8.GetBytes("unrelated owned-destination content"); + File.WriteAllBytes(unrelatedPath, unrelatedBytes); + var verifyError = Assert.Throws( + () => PublisherPackaging.VerifyDestination(destination)); + Assert.Contains("untracked artifact", verifyError.Message, StringComparison.OrdinalIgnoreCase); + File.AppendAllText( + Path.Combine(input, "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"), + "# source content update\n", + new UTF8Encoding(false)); + var secondBuild = workspace.Build(new PublisherBuildOptions(input, state)); + + var error = Assert.Throws(() => PublisherPackaging.Publish( + UnsignedOptions(state, destination, "1.0.0.1", secondBuild.BuildId))); + + Assert.Contains("untracked artifact", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(unrelatedBytes, File.ReadAllBytes(unrelatedPath)); + Assert.Equal(oldIndex, File.ReadAllBytes(Path.Combine(destination, "source2.msix"))); + Assert.Equal(oldMarker, File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json"))); + var verifyErrorAfterRejection = Assert.Throws( + () => PublisherPackaging.VerifyDestination(destination)); + Assert.Contains("untracked artifact", verifyErrorAfterRejection.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Publish_FailedPackagingTool_DoesNotExposeAnIndexOrRecordPublication() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var build = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var where = Path.Combine(Environment.SystemDirectory, "where.exe"); + Assert.True(File.Exists(where), "Windows where.exe is required for the controlled failing-tool case."); + var options = new PublisherPublishOptions( + state, + destination, + PackageName, + Publisher, + "1.0.0.0", + UnsignedDevelopment: true, + MakeAppxPath: where, + BuildId: build.BuildId); + + var error = Assert.Throws(() => PublisherPackaging.Publish(options)); + + Assert.Contains("MSIX packaging failed", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(File.Exists(Path.Combine(destination, "source2.msix"))); + Assert.Null(new PublisherWorkspace(state).Status().LastPublishedBuildId); + + var usableMakeAppx = PublisherPackaging.Doctor().MakeAppxPath; + Assert.False(string.IsNullOrWhiteSpace(usableMakeAppx)); + var retry = PublisherPackaging.Publish(new PublisherPublishOptions( + state, + destination, + PackageName, + Publisher, + "1.0.0.0", + UnsignedDevelopment: true, + MakeAppxPath: usableMakeAppx, + BuildId: build.BuildId)); + Assert.True(retry.Valid); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + Assert.Equal(build.BuildId, new PublisherWorkspace(state).Status().LastPublishedBuildId); + } + + [Fact] + public void Publish_FailedSignatureTool_DoesNotExposeAnIndexOrRecordPublication() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var build = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var makeAppx = PublisherPackaging.Doctor().MakeAppxPath; + Assert.False(string.IsNullOrWhiteSpace(makeAppx), "Windows SDK MakeAppx is required for the signing failure case."); + var where = Path.Combine(Environment.SystemDirectory, "where.exe"); + Assert.True(File.Exists(where), "Windows where.exe is required for the controlled failing-tool case."); + const string password = "safe-test-password"; + var pfx = CreateSigningPfx(Path.Combine(temp.Path, "signer.pfx"), Publisher, password); + const string passwordVariable = "PINGET_PUBLISHER_TEST_PFX_PASSWORD"; + var previousPassword = Environment.GetEnvironmentVariable(passwordVariable); + Environment.SetEnvironmentVariable(passwordVariable, password); + try + { + var options = new PublisherPublishOptions( + state, + destination, + PackageName, + Publisher, + "1.0.0.0", + PfxPath: pfx, + PasswordEnvironmentVariable: passwordVariable, + MakeAppxPath: makeAppx, + SignToolPath: where, + BuildId: build.BuildId); + + var error = Assert.Throws(() => PublisherPackaging.Publish(options)); + + Assert.Contains("MSIX signing failed", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(File.Exists(Path.Combine(destination, "source2.msix"))); + Assert.Null(new PublisherWorkspace(state).Status().LastPublishedBuildId); + + var retry = PublisherPackaging.Publish( + UnsignedOptions(state, destination, "1.0.0.0", build.BuildId)); + Assert.True(retry.Valid); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + Assert.Equal(build.BuildId, new PublisherWorkspace(state).Status().LastPublishedBuildId); + } + finally + { + Environment.SetEnvironmentVariable(passwordVariable, previousPassword); + } + } + + [Fact] + public void Publish_RootIndexPromotionFailure_PreservesPreviousIndexAndOwnershipMarker() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var workspace = new PublisherWorkspace(state); + var firstBuild = workspace.Build(new PublisherBuildOptions(input, state)); + var firstOptions = UnsignedOptions(state, destination, "1.0.0.0", firstBuild.BuildId); + _ = PublisherPackaging.Publish(firstOptions); + var originalIndex = File.ReadAllBytes(Path.Combine(destination, "source2.msix")); + var originalMarker = File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json")); + + var changedManifest = Path.Combine(input, "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + var oldInput = File.ReadAllText(changedManifest); + var updatedInput = oldInput.Replace( + "PackageName: Matrix Package", + "PackageName: Matrix Package Updated", + StringComparison.Ordinal); + Assert.NotEqual(oldInput, updatedInput); + File.WriteAllText(changedManifest, updatedInput, new UTF8Encoding(false)); + var secondBuild = workspace.Build(new PublisherBuildOptions(input, state)); + var next = UnsignedOptions(state, destination, "1.0.0.1", secondBuild.BuildId); + + using (var lockedIndex = new FileStream( + Path.Combine(destination, "source2.msix"), + FileMode.Open, + FileAccess.Read, + FileShare.Read)) + { + Assert.Throws(() => PublisherPackaging.Publish(next)); + Assert.True(lockedIndex.Length > 0); + } + + Assert.Equal(originalIndex, File.ReadAllBytes(Path.Combine(destination, "source2.msix"))); + Assert.Equal(originalMarker, File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json"))); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + Assert.Equal(firstBuild.BuildId, workspace.Status().LastPublishedBuildId); + + var retry = PublisherPackaging.Publish(next); + Assert.True(retry.Valid); + Assert.Equal(secondBuild.BuildId, retry.BuildId); + Assert.NotEqual(originalIndex, File.ReadAllBytes(Path.Combine(destination, "source2.msix"))); + Assert.NotEqual(originalMarker, File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json"))); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + Assert.Equal(secondBuild.BuildId, workspace.Status().LastPublishedBuildId); + } + + [Fact] + public void Publish_DestinationLockSerializesDifferentStateRootsAndAllowsRetry() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var firstState = Path.Combine(temp.Path, "first-state"); + var secondState = Path.Combine(temp.Path, "second-state"); + var destination = Path.Combine(temp.Path, "published"); + var firstBuild = new PublisherWorkspace(firstState) + .Build(new PublisherBuildOptions(input, firstState)); + var secondBuild = new PublisherWorkspace(secondState) + .Build(new PublisherBuildOptions(input, secondState)); + Assert.Equal(firstBuild.BuildId, secondBuild.BuildId); + _ = PublisherPackaging.Publish(UnsignedOptions(firstState, destination, "1.0.0.0", firstBuild.BuildId)); + var oldIndex = File.ReadAllBytes(Path.Combine(destination, "source2.msix")); + var oldMarker = File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json")); + var siblingLock = destination + ".pinget-publisher.lock"; + + using (var heldLock = new FileStream( + siblingLock, + FileMode.OpenOrCreate, + FileAccess.ReadWrite, + FileShare.None)) + { + var error = Assert.ThrowsAny(() => PublisherPackaging.Publish( + UnsignedOptions(secondState, destination, "1.0.0.1", secondBuild.BuildId))); + Assert.Contains("lock", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(oldIndex, File.ReadAllBytes(Path.Combine(destination, "source2.msix"))); + Assert.Equal(oldMarker, File.ReadAllBytes(Path.Combine(destination, ".pinget-publisher.json"))); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + } + + var retry = PublisherPackaging.Publish( + UnsignedOptions(secondState, destination, "1.0.0.1", secondBuild.BuildId)); + Assert.True(retry.Valid); + Assert.Equal(secondBuild.BuildId, retry.BuildId); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + Assert.Equal(secondBuild.BuildId, new PublisherWorkspace(secondState).Status().LastPublishedBuildId); + } + + private static PublisherPublishOptions UnsignedOptions( + string state, + string destination, + string version, + string buildId) => + new( + state, + destination, + PackageName, + Publisher, + version, + UnsignedDevelopment: true, + BuildId: buildId); + + private static void AssertMsixContents(string packagePath, string expectedIndexPath, string version) + { + using var archive = ZipFile.OpenRead(packagePath); + var indexEntry = Assert.IsType(archive.GetEntry("Public/index.db")); + using (var entryStream = indexEntry.Open()) + using (var actualIndex = new MemoryStream()) + { + entryStream.CopyTo(actualIndex); + Assert.Equal(File.ReadAllBytes(expectedIndexPath), actualIndex.ToArray()); + } + + var manifestEntry = Assert.IsType(archive.GetEntry("AppxManifest.xml")); + using var manifestStream = manifestEntry.Open(); + var manifest = XDocument.Load(manifestStream); + XNamespace foundation = "http://schemas.microsoft.com/appx/manifest/foundation/windows10"; + XNamespace uap3 = "http://schemas.microsoft.com/appx/manifest/uap/windows10/3"; + var identity = Assert.IsType(manifest.Root?.Element(foundation + "Identity")); + Assert.Equal(PackageName, (string?)identity.Attribute("Name")); + Assert.Equal(Publisher, (string?)identity.Attribute("Publisher")); + Assert.Equal(version, (string?)identity.Attribute("Version")); + + var extension = Assert.Single(manifest.Descendants(uap3 + "AppExtension")); + Assert.Equal("com.microsoft.winget.source", (string?)extension.Attribute("Name")); + Assert.Equal("IndexDB", (string?)extension.Attribute("Id")); + Assert.Equal("Public", (string?)extension.Attribute("PublicFolder")); + Assert.Equal( + "windows.appExtension", + (string?)extension.Parent?.Attribute("Category")); + } + + private static string CreateSigningPfx(string path, string subject, string password) + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + subject, + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature, critical: true)); + var usages = new OidCollection { new("1.3.6.1.5.5.7.3.3") }; + request.CertificateExtensions.Add(new X509EnhancedKeyUsageExtension(usages, critical: false)); + using var certificate = request.CreateSelfSigned( + DateTimeOffset.UtcNow.AddMinutes(-5), + DateTimeOffset.UtcNow.AddDays(2)); + File.WriteAllBytes(path, certificate.Export(X509ContentType.Pfx, password)); + return path; + } + + private static string CopyFixture(string name, string destination) + { + var source = Path.Combine(AppContext.BaseDirectory, "Fixtures", name); + DirectoryCopy(source, destination); + return destination; + } + + private static void DirectoryCopy(string source, string destination) + { + Directory.CreateDirectory(destination); + foreach (var file in Directory.EnumerateFiles(source)) + File.Copy(file, Path.Combine(destination, Path.GetFileName(file))); + foreach (var directory in Directory.EnumerateDirectories(source)) + DirectoryCopy(directory, Path.Combine(destination, Path.GetFileName(directory))); + } + + private sealed class TempDirectory : IDisposable + { + public TempDirectory() + { + Path = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "pinget-publisher-packaging-tests", + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + if (Directory.Exists(Path)) + Directory.Delete(Path, recursive: true); + } + } +} diff --git a/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherWorkspaceTests.cs b/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherWorkspaceTests.cs new file mode 100644 index 0000000..117da3d --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherWorkspaceTests.cs @@ -0,0 +1,991 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.Data.Sqlite; +using Pinget.Publisher; +using Xunit; + +namespace Devolutions.Pinget.Publisher.Tests; + +public sealed class PublisherWorkspaceTests +{ + [Fact] + public void Build_MultiFileVersionsLocalesAndInstallerMetadata_WritesCompleteV2Snapshot() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + + var build = workspace.Build(new PublisherBuildOptions(input, state)); + + Assert.Equal(1, build.PackageCount); + Assert.Equal(2, build.ManifestCount); + Assert.Equal(0, build.ChangedFiles); + Assert.Equal(0, build.RemovedFiles); + Assert.Equal(build.BuildId, workspace.Status().LastBuildId); + Assert.True(workspace.Verify(build.BuildId).Valid); + + var snapshot = build.SnapshotDirectory; + var indexPath = Path.Combine(snapshot, "Public", "index.db"); + var feed = Path.Combine(snapshot, "feed"); + Assert.True(File.Exists(indexPath)); + using (var snapshotMetadata = JsonDocument.Parse( + File.ReadAllBytes(Path.Combine(snapshot, "snapshot.json")))) + { + Assert.Equal(1, snapshotMetadata.RootElement.GetProperty("schemaVersion").GetInt32()); + Assert.Equal(2, snapshotMetadata.RootElement.GetProperty("artifactFormatVersion").GetInt32()); + var inventory = snapshotMetadata.RootElement.GetProperty("inventory"); + Assert.Equal( + BuildIdForSalt(inventory, "pinget-publisher-state-v2\n"), + build.BuildId); + Assert.NotEqual( + BuildIdForSalt(inventory, "pinget-publisher-state-v1\n"), + build.BuildId); + Assert.Equal( + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(indexPath))).ToLowerInvariant(), + snapshotMetadata.RootElement.GetProperty("indexHash").GetString()); + } + using (var stateDatabase = OpenReadOnly(Path.Combine(state, "publisher.db"))) + { + Assert.Equal("2", ReadMetadataValue(stateDatabase, "artifactFormatVersion")); + Assert.Equal("1", ReadMetadataValue(stateDatabase, "schemaVersion")); + using var command = stateDatabase.CreateCommand(); + command.CommandText = "PRAGMA user_version"; + Assert.Equal(1L, command.ExecuteScalar()); + } + using var database = OpenReadOnly(indexPath); + + Assert.Equal( + ("Pinget.Test.Matrix", "Matrix Package", "1.10.0"), + ReadPackage(database)); + Assert.Equal( + new[] { "Matrix", "matrice", "publisher-conformance" }, + ReadMappedValues(database, "tags2", "tag", "tags2_map", "tag", "Pinget.Test.Matrix")); + Assert.Equal( + new[] { "Matrix", "OtherCmd", "matrix-cli" }, + ReadMappedValues(database, "commands2", "command", "commands2_map", "command", "Pinget.Test.Matrix")); + Assert.Equal( + new[] { "{00112233-4455-6677-8899-AABBCCDDEEFF}" }, + ReadCorrelation(database, "productcodes2", "productcode", "Pinget.Test.Matrix")); + Assert.Equal( + new[] { "{11223344-5566-7788-99AA-BBCCDDEEFF00}" }, + ReadCorrelation(database, "upgradecodes2", "upgradecode", "Pinget.Test.Matrix")); + Assert.Equal( + new[] { "Pinget.Matrix_1234567890abc" }, + ReadCorrelation(database, "pfns2", "pfn", "Pinget.Test.Matrix")); + Assert.Equal( + new[] { "matrice", "matrixdesktop", "matrixpackage" }, + ReadCorrelation(database, "norm_names2", "norm_name", "Pinget.Test.Matrix")); + Assert.Equal( + new[] { "pingetlocalizationlab", "pingettestfixtures" }, + ReadCorrelation(database, "norm_publishers2", "norm_publisher", "Pinget.Test.Matrix")); + + using (var command = database.CreateCommand()) + { + command.CommandText = "SELECT arp_min_version, arp_max_version, hash FROM packages WHERE id=$id"; + command.Parameters.AddWithValue("$id", "Pinget.Test.Matrix"); + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal("1.9.0", reader.GetString(0)); + Assert.Equal("1.10.0", reader.GetString(1)); + var sidecarHash = (byte[])reader[2]; + Assert.Equal(32, sidecarHash.Length); + var hex = Convert.ToHexString(sidecarHash).ToLowerInvariant(); + var relative = $"packages/Pinget.Test.Matrix/{hex[..8]}/versionData.mszyml"; + var sidecar = File.ReadAllBytes(Path.Combine(feed, relative.Replace('/', Path.DirectorySeparatorChar))); + Assert.Equal(hex, Convert.ToHexString(SHA256.HashData(sidecar)).ToLowerInvariant()); + } + + var manifests = Directory.GetFiles(Path.Combine(feed, "manifests"), "*.yaml", SearchOption.AllDirectories); + Assert.Equal(2, manifests.Length); + var englishCurrent = manifests.Single(path => + path.Contains($"{Path.DirectorySeparatorChar}1.10.0{Path.DirectorySeparatorChar}", StringComparison.Ordinal)); + using var mergedCurrent = JsonDocument.Parse(File.ReadAllBytes(englishCurrent)); + var currentRoot = mergedCurrent.RootElement; + Assert.Equal("en-US", currentRoot.GetProperty("PackageLocale").GetString()); + Assert.Equal("Matrix Package", currentRoot.GetProperty("PackageName").GetString()); + Assert.Equal("Pinget Test Fixtures", currentRoot.GetProperty("Publisher").GetString()); + Assert.Equal("1.10.0", currentRoot.GetProperty("PackageVersion").GetString()); + var frenchLocalization = Assert.Single(currentRoot.GetProperty("Localization").EnumerateArray()); + Assert.Equal("fr-FR", frenchLocalization.GetProperty("PackageLocale").GetString()); + Assert.Equal("Matrice", frenchLocalization.GetProperty("PackageName").GetString()); + Assert.Equal("Pinget Localization Lab", frenchLocalization.GetProperty("Publisher").GetString()); + Assert.False(frenchLocalization.TryGetProperty("ManifestType", out _)); + Assert.Equal(2, currentRoot.GetProperty("Installers").GetArrayLength()); + Assert.Equal( + "1111111111111111111111111111111111111111111111111111111111111111", + currentRoot.GetProperty("Installers")[0].GetProperty("InstallerSha256").GetString()); + Assert.Equal( + "2222222222222222222222222222222222222222222222222222222222222222", + currentRoot.GetProperty("Installers")[1].GetProperty("InstallerSha256").GetString()); + var olderEnglishManifest = Directory.GetFiles( + Path.Combine(feed, "manifests", "Pinget.Test.Matrix", "1.9.0", "en-US"), + "*.yaml").Single(); + using var olderManifest = JsonDocument.Parse(File.ReadAllBytes(olderEnglishManifest)); + Assert.Equal( + "0000000000000000000000000000000000000000000000000000000000000000", + olderManifest.RootElement.GetProperty("Installers")[0] + .GetProperty("InstallerSha256").GetString()); + Assert.Equal( + "Pinget.Test.Runtime", + currentRoot.GetProperty("Installers")[0] + .GetProperty("Dependencies") + .GetProperty("PackageDependencies")[0] + .GetProperty("PackageIdentifier") + .GetString()); + Assert.Equal("/quiet", currentRoot.GetProperty("Installers")[0] + .GetProperty("InstallerSwitches").GetProperty("Silent").GetString()); + Assert.Contains( + "portable", + currentRoot.GetProperty("Installers").EnumerateArray() + .Select(installer => installer.GetProperty("InstallerType").GetString())); + } + + [Fact] + public void Build_UsesDeclaredDefaultLocaleWhenAnotherLocaleSortsFirst() + { + using var temp = new TempDirectory(); + var input = Path.Combine(temp.Path, "manifests"); + Directory.CreateDirectory(input); + const string packageId = "Pinget.Test.LocaleOrder"; + WriteLocaleOrderManifests(input, packageId); + var state = Path.Combine(temp.Path, "state"); + + var build = new PublisherWorkspace(state) + .Build(new PublisherBuildOptions(input, state)); + + var versionDirectory = Path.Combine( + build.SnapshotDirectory, "feed", "manifests", packageId, "1.0.0"); + var mergedPaths = Directory.GetFiles(versionDirectory, "*.yaml", SearchOption.AllDirectories); + Assert.Single(mergedPaths); + using var merged = JsonDocument.Parse(File.ReadAllBytes(mergedPaths[0])); + var root = merged.RootElement; + Assert.Equal("en-US", root.GetProperty("PackageLocale").GetString()); + Assert.Equal("Locale Order English", root.GetProperty("PackageName").GetString()); + var localizations = root.GetProperty("Localization"); + var localization = Assert.Single(localizations.EnumerateArray()); + Assert.Equal("de-DE", localization.GetProperty("PackageLocale").GetString()); + Assert.Equal("Locale Reihenfolge", localization.GetProperty("PackageName").GetString()); + Assert.False(localization.TryGetProperty("ManifestType", out _)); + Assert.False(localization.TryGetProperty("ManifestVersion", out _)); + Assert.Equal( + "Pinget.Test.LocaleOrder|Locale Order English|1.0.0", + ReadPackageFromSnapshot(build.SnapshotDirectory)); + Assert.True(new PublisherWorkspace(state).Verify(build.BuildId).Valid); + } + + [Fact] + public void Build_PreservesSchemaStringScalarsInMergedManifest() + { + using var temp = new TempDirectory(); + var input = Path.Combine(temp.Path, "manifests"); + Directory.CreateDirectory(input); + File.WriteAllText(Path.Combine(input, "scalar.yaml"), """ + PackageIdentifier: Pinget.Test.ScalarTypes + PackageVersion: 1.0 + PackageLocale: en-US + Publisher: Pinget Test Fixtures + PackageName: Scalar Types + DisplayVersion: 2024 + SignatureSha256: 0000000000000000000000000000000000000000000000000000000000000000 + GenericNumber: 1.0 + Small: 0.5 + Tiny: 1e-6 + Threshold: 1e-5 + Large: 1e16 + NegativeZero: -0.0 + MaxUInt: 18446744073709551615 + Installers: [{Architecture: x64, InstallerType: exe, InstallerUrl: 'https://example.invalid/scalar.exe', InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000}] + ManifestType: singleton + ManifestVersion: 1.9.0 + """, new UTF8Encoding(false)); + var state = Path.Combine(temp.Path, "state"); + + var build = new PublisherWorkspace(state) + .Build(new PublisherBuildOptions(input, state)); + + var mergedPath = Directory.GetFiles( + Path.Combine(build.SnapshotDirectory, "feed", "manifests"), + "*.yaml", + SearchOption.AllDirectories).Single(); + using var merged = JsonDocument.Parse(File.ReadAllBytes(mergedPath)); + var root = merged.RootElement; + Assert.Equal("1.0", root.GetProperty("PackageVersion").GetString()); + Assert.Equal("2024", root.GetProperty("DisplayVersion").GetString()); + Assert.Equal( + "0000000000000000000000000000000000000000000000000000000000000000", + root.GetProperty("SignatureSha256").GetString()); + Assert.Equal( + "0000000000000000000000000000000000000000000000000000000000000000", + root.GetProperty("Installers")[0].GetProperty("InstallerSha256").GetString()); + Assert.Equal(JsonValueKind.Number, root.GetProperty("GenericNumber").ValueKind); + Assert.Equal(1.0, root.GetProperty("GenericNumber").GetDouble()); + Assert.Equal(0.5, root.GetProperty("Small").GetDouble()); + Assert.Equal(1e-6, root.GetProperty("Tiny").GetDouble()); + Assert.Equal(1e-5, root.GetProperty("Threshold").GetDouble()); + Assert.Equal(1e16, root.GetProperty("Large").GetDouble()); + Assert.Equal(-0.0, root.GetProperty("NegativeZero").GetDouble()); + Assert.Equal(ulong.MaxValue, root.GetProperty("MaxUInt").GetUInt64()); + } + + [Theory] + [InlineData("1.0.0", "1.0.1", "1.0.1")] + [InlineData("1.0", "1.0.0-alpha", "1.0")] + public void Build_VersionOrderingHandlesMissingComponentsAndSuffixes( + string firstVersion, + string secondVersion, + string expectedLatestVersion) + { + using var temp = new TempDirectory(); + var input = Path.Combine(temp.Path, "manifests"); + Directory.CreateDirectory(input); + const string packageId = "Pinget.Test.VersionOrdering"; + File.WriteAllText( + Path.Combine(input, "first.yaml"), + SingletonManifest(packageId, firstVersion), + new UTF8Encoding(false)); + File.WriteAllText( + Path.Combine(input, "second.yaml"), + SingletonManifest(packageId, secondVersion), + new UTF8Encoding(false)); + var state = Path.Combine(temp.Path, "state"); + + var build = new PublisherWorkspace(state) + .Build(new PublisherBuildOptions(input, state)); + + using var database = OpenReadOnly( + Path.Combine(build.SnapshotDirectory, "Public", "index.db")); + Assert.Equal( + (packageId, packageId, expectedLatestVersion), + ReadPackage(database)); + } + + [Fact] + public void Build_EquivalentInputTreesHaveSameBuildIdentity() + { + using var temp = new TempDirectory(); + var firstInput = CopyFixture("multi", Path.Combine(temp.Path, "first")); + var secondInput = CopyFixture("multi", Path.Combine(temp.Path, "second")); + var firstState = Path.Combine(temp.Path, "state-a"); + var secondState = Path.Combine(temp.Path, "state-b"); + + var first = new PublisherWorkspace(firstState) + .Build(new PublisherBuildOptions(firstInput, firstState)); + var second = new PublisherWorkspace(secondState) + .Build(new PublisherBuildOptions(secondInput, secondState)); + + Assert.Equal(first.BuildId, second.BuildId); + Assert.Equal(ReadPackageFromSnapshot(first.SnapshotDirectory), ReadPackageFromSnapshot(second.SnapshotDirectory)); + } + + [Fact] + public void Diff_RawMetadataEditsAdditionsAndRemovals_AreReportedByRelativePath() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var first = workspace.Build(new PublisherBuildOptions(input, state)); + Assert.Empty(workspace.Diff(input).Added); + Assert.Empty(workspace.Diff(input).Changed); + Assert.Empty(workspace.Diff(input).Removed); + + var changed = Path.Combine(input, "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + File.AppendAllText(changed, "# raw-byte change\n", new UTF8Encoding(false)); + File.WriteAllText(Path.Combine(input, "Pinget.Test.Added.yaml"), SingletonManifest("Pinget.Test.Added", "1.0.0")); + var removed = Path.Combine(input, "Pinget.Test.Matrix_1.9.0.yaml"); + File.Delete(removed); + + var diff = workspace.Diff(input); + Assert.False(diff.Published); + Assert.Equal(new[] { "Pinget.Test.Added.yaml" }, diff.Added); + Assert.Equal(new[] { "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml" }, diff.Changed); + Assert.Equal(new[] { "Pinget.Test.Matrix_1.9.0.yaml" }, diff.Removed); + + var second = workspace.Build(new PublisherBuildOptions(input, state)); + Assert.NotEqual(first.BuildId, second.BuildId); + Assert.True(workspace.Verify(second.BuildId).Valid); + Assert.Equal(2, workspace.Status().BuildCount); + } + + [Theory] + [InlineData("missing-version", "requires exactly one version")] + [InlineData("duplicate-version", "requires exactly one version")] + [InlineData("unsupported-schema", "Unsupported ManifestVersion")] + [InlineData("invalid-version", "Unsafe PackageVersion")] + [InlineData("unsafe-identifier", "Unsafe PackageIdentifier")] + [InlineData("duplicate-locale", "Duplicate locale")] + [InlineData("duplicate-singleton", "Duplicate singleton")] + [InlineData("missing-installer", "installer")] + [InlineData("empty-default-locale", "DefaultLocale")] + [InlineData("duplicate-default-locale", "duplicate")] + [InlineData("missing-singleton-locale", "PackageLocale")] + public void Build_InvalidManifestPartition_IsRejected(string scenario, string expectedMessage) + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + ApplyInvalidManifestMutation(input, scenario); + var state = Path.Combine(temp.Path, "state"); + + var error = Assert.Throws(() => + new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state))); + + Assert.Contains(expectedMessage, error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(File.Exists(Path.Combine(state, "publisher.db"))); + Assert.Empty(Directory.Exists(Path.Combine(state, "builds")) + ? Directory.GetDirectories(Path.Combine(state, "builds")) + : []); + } + + [Fact] + public void Build_RejectsCaseConflictingPackageIdentity() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + File.WriteAllText( + Path.Combine(input, "case-conflict.yaml"), + SingletonManifest("pinget.test.matrix", "2.0.0"), + new UTF8Encoding(false)); + var state = Path.Combine(temp.Path, "state"); + + var error = Assert.Throws(() => + new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state))); + + Assert.Contains("casing", error.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ReadOnlyOperations_MissingState_DoNotInitializeIt() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "not-created"); + var workspace = new PublisherWorkspace(state); + + Assert.Throws(() => workspace.Status()); + Assert.Throws(() => workspace.Diff(input)); + + Assert.False(Directory.Exists(state)); + } + + [Fact] + public void Build_FutureUserVersionIsRejectedWithoutChangingTheStateDatabase() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + _ = workspace.Build(new PublisherBuildOptions(input, state)); + using (var database = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = Path.Combine(state, "publisher.db"), + Mode = SqliteOpenMode.ReadWrite, + Pooling = false, + }.ToString())) + { + database.Open(); + using var command = database.CreateCommand(); + command.CommandText = "PRAGMA user_version=999;"; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + var stateBytes = File.ReadAllBytes(Path.Combine(state, "publisher.db")); + + var error = Assert.ThrowsAny(() => + workspace.Build(new PublisherBuildOptions(input, state))); + + Assert.False(string.IsNullOrWhiteSpace(error.Message)); + Assert.Equal(stateBytes, File.ReadAllBytes(Path.Combine(state, "publisher.db"))); + } + + [Theory] + [InlineData("missing")] + [InlineData("1")] + [InlineData("999")] + public void Build_UnsupportedArtifactFormatStateIsRejectedWithoutMutation(string formatVersion) + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + _ = workspace.Build(new PublisherBuildOptions(input, state)); + using (var database = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = Path.Combine(state, "publisher.db"), + Mode = SqliteOpenMode.ReadWrite, + Pooling = false, + }.ToString())) + { + database.Open(); + using var command = database.CreateCommand(); + command.CommandText = formatVersion == "missing" + ? "DELETE FROM metadata WHERE name='artifactFormatVersion';" + : "UPDATE metadata SET value=$value WHERE name='artifactFormatVersion';"; + if (formatVersion != "missing") + command.Parameters.AddWithValue("$value", formatVersion); + Assert.Equal(1, command.ExecuteNonQuery()); + } + SqliteConnection.ClearAllPools(); + var databasePath = Path.Combine(state, "publisher.db"); + var legacyStateBytes = File.ReadAllBytes(databasePath); + + var error = Assert.ThrowsAny(() => + workspace.Build(new PublisherBuildOptions(input, state))); + + Assert.Contains("artifactFormatVersion", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("--state", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(legacyStateBytes, File.ReadAllBytes(databasePath)); + } + + [Theory] + [InlineData("missing")] + [InlineData("1")] + [InlineData("999")] + public void Verify_UnsupportedArtifactFormatSnapshotIsRejectedWithoutMutation(string formatVersion) + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + var metadataPath = Path.Combine(build.SnapshotDirectory, "snapshot.json"); + var metadata = JsonNode.Parse(File.ReadAllText(metadataPath))!.AsObject(); + Assert.Equal(2, metadata["artifactFormatVersion"]!.GetValue()); + if (formatVersion == "missing") + metadata.Remove("artifactFormatVersion"); + else + metadata["artifactFormatVersion"] = int.Parse(formatVersion, System.Globalization.CultureInfo.InvariantCulture); + File.WriteAllText(metadataPath, metadata.ToJsonString(), new UTF8Encoding(false)); + var legacySnapshotBytes = File.ReadAllBytes(metadataPath); + + var error = Assert.ThrowsAny(() => workspace.Verify(build.BuildId)); + + Assert.Contains("format", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(legacySnapshotBytes, File.ReadAllBytes(metadataPath)); + } + + [Fact] + public void Build_ForeignSqliteDatabaseIsRejectedWithoutInitializationOrMutation() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + Directory.CreateDirectory(state); + var databasePath = Path.Combine(state, "publisher.db"); + using (var database = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = databasePath, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false, + }.ToString())) + { + database.Open(); + using var command = database.CreateCommand(); + command.CommandText = "CREATE TABLE unrelated_owner(value TEXT); INSERT INTO unrelated_owner VALUES('preserve-me');"; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + var originalDatabase = File.ReadAllBytes(databasePath); + + Assert.ThrowsAny(() => + new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state))); + + Assert.Equal(originalDatabase, File.ReadAllBytes(databasePath)); + Assert.Empty(Directory.Exists(Path.Combine(state, "builds")) + ? Directory.GetDirectories(Path.Combine(state, "builds")) + : []); + } + + [Fact] + public void Build_StaleStagingDirectoryDoesNotBlockAValidSnapshot() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var staleStaging = Path.Combine(state, "builds", ".staging-stale-owned-attempt"); + Directory.CreateDirectory(staleStaging); + var staleMarker = Path.Combine(staleStaging, "preserve.txt"); + File.WriteAllText(staleMarker, "not owned by this invocation", new UTF8Encoding(false)); + + var build = new PublisherWorkspace(state) + .Build(new PublisherBuildOptions(input, state)); + + Assert.True(new PublisherWorkspace(state).Verify(build.BuildId).Valid); + Assert.Equal("not owned by this invocation", File.ReadAllText(staleMarker)); + Assert.True(Directory.Exists(staleStaging)); + } + + [Fact] + public void Verify_UnrecordedCopiedSnapshotIsRejected() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + var copiedId = new string('f', 64); + var copiedSnapshot = Path.Combine(state, "builds", copiedId); + DirectoryCopy(build.SnapshotDirectory, copiedSnapshot); + var metadataPath = Path.Combine(copiedSnapshot, "snapshot.json"); + var metadata = File.ReadAllText(metadataPath); + File.WriteAllText( + metadataPath, + metadata.Replace(build.BuildId, copiedId, StringComparison.Ordinal), + new UTF8Encoding(false)); + + var error = Assert.ThrowsAny(() => workspace.Verify(copiedId)); + + Assert.Contains("build", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.True(workspace.Verify(build.BuildId).Valid); + } + + [Theory] + [InlineData("schema")] + [InlineData("build-id")] + [InlineData("inventory")] + public void Verify_SnapshotSchemaIdentityAndInventoryTamperingAreRejected(string mutation) + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + var snapshotMetadataPath = Path.Combine(build.SnapshotDirectory, "snapshot.json"); + var snapshot = JsonNode.Parse(File.ReadAllText(snapshotMetadataPath))!.AsObject(); + switch (mutation) + { + case "schema": + snapshot["schemaVersion"] = 99; + break; + case "build-id": + snapshot["buildId"] = new string('f', 64); + break; + case "inventory": + var inventory = snapshot["inventory"]!.AsObject(); + var first = inventory.First(); + inventory[first.Key] = new string('0', 64); + break; + } + File.WriteAllText(snapshotMetadataPath, snapshot.ToJsonString(), new UTF8Encoding(false)); + + Assert.ThrowsAny(() => workspace.Verify(build.BuildId)); + } + + [Fact] + public void Verify_UntrackedFeedArtifactIsRejected() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + File.WriteAllText( + Path.Combine(build.SnapshotDirectory, "feed", "unexpected.txt"), + "not listed in snapshot inventory", + new UTF8Encoding(false)); + + var error = Assert.ThrowsAny(() => workspace.Verify(build.BuildId)); + + Assert.True( + error.Message.Contains("inventory", StringComparison.OrdinalIgnoreCase) || + error.Message.Contains("artifact", StringComparison.OrdinalIgnoreCase), + $"Unexpected untracked-artifact diagnostic: {error.Message}"); + } + + [Fact] + public void ReadOnlyOperations_RejectReparsePointsInInputAndStateRoots() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + var inputLink = Path.Combine(temp.Path, "manifests-link"); + var stateLink = Path.Combine(temp.Path, "state-link"); + Directory.CreateSymbolicLink(inputLink, input); + Directory.CreateSymbolicLink(stateLink, state); + try + { + Assert.ThrowsAny(() => workspace.Diff(inputLink)); + Assert.ThrowsAny(() => new PublisherWorkspace(stateLink).Status()); + Assert.ThrowsAny(() => new PublisherWorkspace(stateLink).Verify(build.BuildId)); + } + finally + { + Directory.Delete(inputLink); + Directory.Delete(stateLink); + } + } + + [Fact] + public void Diff_RejectsInputRootNestedInsidePublisherState() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var workspace = new PublisherWorkspace(state); + var build = workspace.Build(new PublisherBuildOptions(input, state)); + + var error = Assert.ThrowsAny(() => + workspace.Diff(Path.Combine(state, "builds", build.BuildId, "feed"))); + + Assert.Contains("overlap", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.True(workspace.Verify(build.BuildId).Valid); + } + + [Fact] + public void Build_RejectsReparsePointWithinManifestRoot() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var external = Path.Combine(temp.Path, "external"); + Directory.CreateDirectory(external); + var link = Path.Combine(input, "linked"); + Directory.CreateSymbolicLink(link, external); + try + { + var error = Assert.Throws(() => + new PublisherWorkspace(Path.Combine(temp.Path, "state")) + .Build(new PublisherBuildOptions(input, Path.Combine(temp.Path, "state")))); + Assert.Contains("reparse", error.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + Directory.Delete(link); + } + } + + [Fact] + public void Build_InputAndStateRootsMayNotOverlap() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(input, "publisher-state"); + + var error = Assert.Throws(() => + new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state))); + + Assert.Contains("overlap", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(Directory.Exists(state)); + } + + [Fact] + public void Build_ExistingWriterLock_PreventsConcurrentMutation() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + Directory.CreateDirectory(state); + using var heldLock = new FileStream( + Path.Combine(state, ".publisher.lock"), + FileMode.OpenOrCreate, + FileAccess.ReadWrite, + FileShare.None); + + var error = Assert.Throws(() => + new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state))); + + Assert.Contains("lock", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(File.Exists(Path.Combine(state, "publisher.db"))); + } + + [Theory] + [InlineData("snapshot")] + [InlineData("manifest")] + [InlineData("sidecar")] + [InlineData("index")] + public void Verify_CorruptedSnapshotComponent_IsRejected(string component) + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var build = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var snapshot = build.SnapshotDirectory; + + switch (component) + { + case "snapshot": + File.WriteAllText(Path.Combine(snapshot, "snapshot.json"), "{not-json"); + break; + case "manifest": + MutateFirstFile(Path.Combine(snapshot, "feed", "manifests"), bytes => bytes.Concat(new byte[] { 0x20 }).ToArray()); + break; + case "sidecar": + MutateFirstFile(Path.Combine(snapshot, "feed", "packages"), bytes => bytes.Concat(new byte[] { 0x20 }).ToArray()); + break; + case "index": + File.WriteAllBytes(Path.Combine(snapshot, "Public", "index.db"), new byte[] { 0x01, 0x02, 0x03 }); + break; + } + + Assert.ThrowsAny(() => new PublisherWorkspace(state).Verify(build.BuildId)); + } + + [Fact] + public void Status_CorruptStateDatabase_IsRejectedWithoutReinitializingIt() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var database = Path.Combine(state, "publisher.db"); + File.WriteAllBytes(database, Encoding.ASCII.GetBytes("not a sqlite database")); + + Assert.ThrowsAny(() => new PublisherWorkspace(state).Status()); + Assert.Equal("not a sqlite database", Encoding.ASCII.GetString(File.ReadAllBytes(database))); + } + + private static string ReadPackageFromSnapshot(string snapshot) + { + using var database = OpenReadOnly(Path.Combine(snapshot, "Public", "index.db")); + var (id, name, version) = ReadPackage(database); + return $"{id}|{name}|{version}"; + } + + private static string ReadMetadataValue(SqliteConnection database, string key) + { + using var command = database.CreateCommand(); + command.CommandText = "SELECT value FROM metadata WHERE name=$name"; + command.Parameters.AddWithValue("$name", key); + return command.ExecuteScalar() as string + ?? throw new InvalidOperationException($"Missing publisher state metadata '{key}'."); + } + + private static string BuildIdForSalt(JsonElement inventory, string salt) + { + var inventoryJson = "{" + + string.Join( + ",", + inventory.EnumerateObject() + .OrderBy(property => property.Name, StringComparer.Ordinal) + .Select(property => + $"{JsonSerializer.Serialize(property.Name)}:{JsonSerializer.Serialize(property.Value.GetString() ?? throw new InvalidOperationException("Inventory values must be strings."))}")) + + "}"; + return Convert.ToHexString( + SHA256.HashData(Encoding.UTF8.GetBytes(salt + inventoryJson))) + .ToLowerInvariant(); + } + + private static (string Id, string Name, string Version) ReadPackage(SqliteConnection database) + { + using var command = database.CreateCommand(); + command.CommandText = "SELECT id, name, latest_version FROM packages"; + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + var package = (reader.GetString(0), reader.GetString(1), reader.GetString(2)); + Assert.False(reader.Read()); + return package; + } + + private static string[] ReadMappedValues( + SqliteConnection database, + string valueTable, + string valueColumn, + string mapTable, + string mapColumn, + string packageId) + { + using var command = database.CreateCommand(); + command.CommandText = $""" + SELECT valuesTable.{valueColumn} + FROM {valueTable} AS valuesTable + JOIN {mapTable} AS valueMap ON valueMap.{mapColumn}=valuesTable.rowid + JOIN packages AS package ON package.rowid=valueMap.package + WHERE package.id=$id + ORDER BY valuesTable.{valueColumn} + """; + command.Parameters.AddWithValue("$id", packageId); + using var reader = command.ExecuteReader(); + var values = new List(); + while (reader.Read()) + values.Add(reader.GetString(0)); + return values.ToArray(); + } + + private static string[] ReadCorrelation(SqliteConnection database, string table, string column, string packageId) + { + using var command = database.CreateCommand(); + command.CommandText = $"SELECT {column} FROM {table} AS correlation JOIN packages AS package ON package.rowid=correlation.package WHERE package.id=$id ORDER BY {column}"; + command.Parameters.AddWithValue("$id", packageId); + using var reader = command.ExecuteReader(); + var values = new List(); + while (reader.Read()) + values.Add(reader.GetString(0)); + return values.ToArray(); + } + + private static SqliteConnection OpenReadOnly(string path) + { + var connection = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = SqliteOpenMode.ReadOnly, + Pooling = false, + }.ToString()); + connection.Open(); + return connection; + } + + private static string CopyFixture(string name, string destination) + { + var source = Path.Combine(AppContext.BaseDirectory, "Fixtures", name); + DirectoryCopy(source, destination); + return destination; + } + + private static void DirectoryCopy(string source, string destination) + { + Directory.CreateDirectory(destination); + foreach (var file in Directory.EnumerateFiles(source)) + File.Copy(file, Path.Combine(destination, Path.GetFileName(file))); + foreach (var directory in Directory.EnumerateDirectories(source)) + DirectoryCopy(directory, Path.Combine(destination, Path.GetFileName(directory))); + } + + private static void ApplyInvalidManifestMutation(string input, string scenario) + { + var version = Path.Combine(input, "Pinget.Test.Matrix_1.10.0.version.yaml"); + var french = Path.Combine(input, "Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml"); + switch (scenario) + { + case "missing-version": + File.Delete(version); + break; + case "duplicate-version": + File.Copy(version, Path.Combine(input, "duplicate-version.yaml")); + break; + case "unsupported-schema": + File.WriteAllText(version, File.ReadAllText(version).Replace("ManifestVersion: 1.9.0", "ManifestVersion: 9.9.9", StringComparison.Ordinal)); + break; + case "invalid-version": + File.WriteAllText(version, File.ReadAllText(version).Replace("PackageVersion: 1.10.0", "PackageVersion: ../escape", StringComparison.Ordinal)); + break; + case "unsafe-identifier": + File.WriteAllText(version, File.ReadAllText(version).Replace("Pinget.Test.Matrix", "../escape", StringComparison.Ordinal)); + break; + case "duplicate-locale": + File.Copy(french, Path.Combine(input, "duplicate-fr.yaml")); + break; + case "duplicate-singleton": + File.Copy(Path.Combine(input, "Pinget.Test.Matrix_1.9.0.yaml"), Path.Combine(input, "duplicate-singleton.yaml")); + break; + case "missing-installer": + File.Delete(Path.Combine(input, "Pinget.Test.Matrix_1.10.0.installer.yaml")); + break; + case "empty-default-locale": + File.WriteAllText(version, File.ReadAllText(version).Replace( + "DefaultLocale: en-US", "DefaultLocale: ''", StringComparison.Ordinal)); + break; + case "duplicate-default-locale": + File.Copy( + Path.Combine(input, "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"), + Path.Combine(input, "duplicate-default-locale.yaml")); + File.WriteAllText( + Path.Combine(input, "duplicate-default-locale.yaml"), + File.ReadAllText(Path.Combine(input, "duplicate-default-locale.yaml")) + .Replace("PackageLocale: en-US", "PackageLocale: en-us", StringComparison.Ordinal)); + break; + case "missing-singleton-locale": + var missingLocale = Regex.Replace( + SingletonManifest("Pinget.Test.MissingLocale", "1.0.0"), + @"(?m)^PackageLocale: en-US\r?\n", + "", + RegexOptions.CultureInvariant); + Assert.DoesNotContain("PackageLocale:", missingLocale, StringComparison.Ordinal); + File.WriteAllText( + Path.Combine(input, "missing-locale.yaml"), + missingLocale); + break; + default: + throw new ArgumentOutOfRangeException(nameof(scenario)); + } + } + + private static string SingletonManifest(string id, string version) => $$""" + PackageIdentifier: {{id}} + PackageVersion: {{version}} + PackageLocale: en-US + Publisher: Pinget Test Fixtures + PackageName: {{id}} + License: MIT + ShortDescription: temporary fixture + Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://example.invalid/fixture.exe + InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 + ManifestType: singleton + ManifestVersion: 1.9.0 + """; + + private static void WriteLocaleOrderManifests(string input, string id) + { + foreach (var (suffix, locale, name, type) in new[] + { + ("locale.en-US", "en-US", "Locale Order English", "defaultLocale"), + ("locale.de-DE", "de-DE", "Locale Reihenfolge", "locale"), + }) + { + File.WriteAllText(Path.Combine(input, $"{id}_1.0.0.{suffix}.yaml"), $$""" + PackageIdentifier: {{id}} + PackageVersion: 1.0.0 + PackageLocale: {{locale}} + Publisher: Pinget Test Fixtures + PackageName: {{name}} + License: MIT + ShortDescription: locale-order fixture + ManifestType: {{type}} + ManifestVersion: 1.9.0 + """, new UTF8Encoding(false)); + } + + File.WriteAllText(Path.Combine(input, $"{id}_1.0.0.version.yaml"), $$""" + PackageIdentifier: {{id}} + PackageVersion: 1.0.0 + DefaultLocale: en-US + ManifestType: version + ManifestVersion: 1.9.0 + """, new UTF8Encoding(false)); + File.WriteAllText(Path.Combine(input, $"{id}_1.0.0.installer.yaml"), $$""" + PackageIdentifier: {{id}} + PackageVersion: 1.0.0 + Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://example.invalid/locale-order.exe + InstallerSha256: 1111111111111111111111111111111111111111111111111111111111111111 + ManifestType: installer + ManifestVersion: 1.9.0 + """, new UTF8Encoding(false)); + } + + private static void MutateFirstFile(string directory, Func mutation) + { + var file = Directory.GetFiles(directory, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal).First(); + File.WriteAllBytes(file, mutation(File.ReadAllBytes(file))); + } + + private sealed class TempDirectory : IDisposable + { + public TempDirectory() + { + Path = System.IO.Path.Combine( + System.IO.Path.GetTempPath(), + "pinget-publisher-tests", + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(Path)) + Directory.Delete(Path, recursive: true); + } + } +} diff --git a/dotnet/src/Devolutions.Pinget.Publisher/Devolutions.Pinget.Publisher.csproj b/dotnet/src/Devolutions.Pinget.Publisher/Devolutions.Pinget.Publisher.csproj new file mode 100644 index 0000000..f67f498 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/Devolutions.Pinget.Publisher.csproj @@ -0,0 +1,27 @@ + + + + net8.0;net10.0 + enable + enable + true + Devolutions.Pinget.Publisher + Pinget.Publisher + Devolutions.Pinget.Publisher + Native WinGet v2 preindexed source publisher for Pinget. + + + + + + + + + + + + + + + + diff --git a/dotnet/src/Devolutions.Pinget.Publisher/ManifestScanner.cs b/dotnet/src/Devolutions.Pinget.Publisher/ManifestScanner.cs new file mode 100644 index 0000000..0814fc0 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/ManifestScanner.cs @@ -0,0 +1,715 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Encodings.Web; +using System.Text.RegularExpressions; +using YamlDotNet.RepresentationModel; + +namespace Pinget.Publisher; + +internal sealed record ManifestFile(string RelativePath, string Hash); + +internal sealed record MergedManifest( + string PackageId, + string Version, + string Locale, + string DefaultLocale, + string RelativePath, + byte[] Content, + string Hash, + JsonObject Model); + +internal sealed record IndexedPackage( + string Id, + string Name, + string? Moniker, + string LatestVersion, + IReadOnlyList Manifests, + IReadOnlyList Tags, + IReadOnlyList Commands, + IReadOnlyList Pfns, + IReadOnlyList ProductCodes, + IReadOnlyList UpgradeCodes, + IReadOnlyList NormalizedNames, + IReadOnlyList NormalizedPublishers, + string? ArpMinimumVersion, + string? ArpMaximumVersion, + byte[] VersionData, + byte[] CompressedVersionData, + byte[] PackageHash); + +internal sealed record ScanResult( + IReadOnlyDictionary Inventory, + IReadOnlyList Packages, + IReadOnlyList Manifests); + +internal static partial class ManifestScanner +{ + private static readonly HashSet SupportedSchemaVersions = new(StringComparer.Ordinal) + { + "1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", + "1.6.0", "1.7.0", "1.8.0", "1.9.0", "1.10.0", + }; + + private static readonly HashSet DocumentTypes = new(StringComparer.Ordinal) + { + "singleton", "version", "installer", "defaultLocale", "locale", + }; + + public static ScanResult Scan(string inputDirectory, bool generateArtifacts = true) + { + var root = Path.GetFullPath(inputDirectory); + if (!Directory.Exists(root)) + throw new PublisherException($"Manifest directory does not exist: {root}"); + RejectReparsePoint(root); + + var files = EnumerateYamlFiles(root); + if (files.Count == 0) + throw new PublisherException($"No YAML manifests were found under '{root}'."); + + var inventory = new SortedDictionary(StringComparer.Ordinal); + var documents = new Dictionary<(string Id, string Version), List>(); + var casing = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var file in files) + { + var relativePath = Path.GetRelativePath(root, file).Replace('\\', '/'); + var bytes = File.ReadAllBytes(file); + if (bytes.Length > 8 * 1024 * 1024) + throw new PublisherException($"Manifest exceeds the 8 MiB limit: {relativePath}"); + var rawHash = Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes)).ToLowerInvariant(); + inventory.Add(relativePath, rawHash); + + var model = ParseYaml(bytes, relativePath); + var id = RequiredString(model, "PackageIdentifier", relativePath); + var version = RequiredString(model, "PackageVersion", relativePath); + var type = RequiredString(model, "ManifestType", relativePath); + var schemaVersion = RequiredString(model, "ManifestVersion", relativePath); + if (!IdentifierPattern().IsMatch(id)) + throw new PublisherException($"Unsafe PackageIdentifier '{id}' in {relativePath}."); + if (casing.TryGetValue(id, out var existingId) && !string.Equals(existingId, id, StringComparison.Ordinal)) + throw new PublisherException($"PackageIdentifier differs only by casing: '{existingId}' and '{id}'."); + casing[id] = id; + if (!VersionPattern().IsMatch(version)) + throw new PublisherException($"Unsafe PackageVersion '{version}' in {relativePath}."); + if (!DocumentTypes.Contains(type)) + throw new PublisherException($"Unsupported ManifestType '{type}' in {relativePath}."); + if (!SupportedSchemaVersions.Contains(schemaVersion)) + throw new PublisherException($"Unsupported ManifestVersion '{schemaVersion}' in {relativePath}; supported versions are {string.Join(", ", SupportedSchemaVersions)}."); + + string? locale = OptionalString(model, "PackageLocale"); + if (type is "defaultLocale" or "locale") + { + if (string.IsNullOrWhiteSpace(locale) || !LocalePattern().IsMatch(locale)) + throw new PublisherException($"Invalid or missing PackageLocale in {relativePath}."); + } + + var key = (id, version); + if (!documents.TryGetValue(key, out var group)) + documents.Add(key, group = []); + group.Add(new ManifestDocument(type, locale, relativePath, model)); + } + + var merged = MergeDocuments(documents); + RejectOutputCollisions(merged); + foreach (var group in merged.GroupBy(m => m.PackageId, StringComparer.OrdinalIgnoreCase)) + _ = ValidateVersions(group); + return generateArtifacts ? BuildPackages(inventory, merged) : new ScanResult(inventory, [], merged); + } + + private static List EnumerateYamlFiles(string root) + { + var results = new List(); + var pending = new Stack(); + pending.Push(root); + while (pending.TryPop(out var directory)) + { + RejectReparsePoint(directory); + foreach (var entry in Directory.EnumerateFileSystemEntries(directory)) + { + RejectReparsePoint(entry); + if (Directory.Exists(entry)) + pending.Push(entry); + else if (Path.GetExtension(entry).Equals(".yaml", StringComparison.OrdinalIgnoreCase) + || Path.GetExtension(entry).Equals(".yml", StringComparison.OrdinalIgnoreCase)) + results.Add(entry); + } + } + results.Sort(StringComparer.Ordinal); + return results; + } + + private static void RejectReparsePoint(string path) + { + if ((File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Symbolic links and reparse points are not allowed in publisher paths: {path}"); + } + + private static JsonObject ParseYaml(byte[] bytes, string path) + { + try + { + using var reader = new StringReader(new UTF8Encoding(false, true).GetString(bytes)); + var stream = new YamlStream(); + stream.Load(reader); + if (stream.Documents.Count != 1 || stream.Documents[0].RootNode is not YamlMappingNode mapping) + throw new PublisherException($"Expected one YAML mapping document in {path}."); + return (JsonObject)ConvertNode(mapping, 0); + } + catch (PublisherException) + { + throw; + } + catch (Exception ex) + { + throw new PublisherException($"Could not parse YAML manifest '{path}': {ex.Message}", ex); + } + } + + private static JsonNode ConvertNode(YamlNode node, int depth) + { + if (depth > 100) + throw new PublisherException("YAML nesting depth exceeds the 100-level limit."); + return node switch + { + YamlMappingNode mapping => ConvertMapping(mapping, depth + 1), + YamlSequenceNode sequence => new JsonArray(sequence.Children.Select(child => ConvertNode(child, depth + 1)).ToArray()), + YamlScalarNode scalar => ConvertScalar(scalar), + _ => throw new PublisherException($"Unsupported YAML node type: {node.GetType().Name}"), + }; + } + + private static JsonObject ConvertMapping(YamlMappingNode mapping, int depth) + { + var result = new JsonObject(); + foreach (var pair in mapping.Children) + { + if (pair.Key is not YamlScalarNode key || string.IsNullOrEmpty(key.Value)) + throw new PublisherException("Manifest mapping keys must be non-empty strings."); + if (result.ContainsKey(key.Value)) + throw new PublisherException($"Duplicate YAML mapping key '{key.Value}'."); + result.Add(key.Value, IsStringField(key.Value) + ? ConvertStringNode(pair.Value, depth) : ConvertNode(pair.Value, depth)); + } + + return result; + } + + private static JsonNode ConvertStringNode(YamlNode node, int depth) + { + if (depth > 100) + throw new PublisherException("YAML nesting depth exceeds the 100-level limit."); + return node switch + { + YamlScalarNode scalar => JsonValue.Create(scalar.Value ?? string.Empty)!, + YamlSequenceNode sequence => new JsonArray(sequence.Children.Select(child => ConvertStringNode(child, depth + 1)).ToArray()), + _ => ConvertNode(node, depth), + }; + } + + private static bool IsStringField(string key) => + key.EndsWith("Sha256", StringComparison.Ordinal) || key.EndsWith("Url", StringComparison.Ordinal) + || key is "PackageIdentifier" or "PackageVersion" or "ManifestVersion" or "ManifestType" + or "PackageLocale" or "DefaultLocale" or "PackageName" or "Publisher" or "Moniker" + or "DisplayName" or "DisplayVersion" or "MinimumVersion" or "MinimumOSVersion" + or "ProductCode" or "UpgradeCode" or "PackageFamilyName" or "InstallerLocale" + or "Architecture" or "InstallerType" or "Scope" or "FileType" or "IconFileType" + or "License" or "Copyright" or "ShortDescription" or "Description" or "ReleaseNotes" + or "ReleaseDate" or "RelativeFilePath" or "PortableCommandAlias" or "Commands" or "Tags" + or "FileExtensions" or "Protocols" or "Capabilities" or "RestrictedCapabilities" + or "InstallerSwitches" or "InstallLocation" + or "InstallModes" or "Platform" or "UpgradeBehavior" or "NestedInstallerType" + or "IconResolution" or "IconTheme" or "AuthenticationType" or "Argument" + or "Command" or "Custom" or "Silent" or "SilentWithProgress" or "Interactive" + or "Log" or "Repair" or "Uninstall" or "Update" or "Channel" + or "WindowsFeatures" or "WindowsLibraries" or "ExternalDependencies" or "DocumentLabel" + or "InvocationParameter" or "ReturnResponse" or "AllowedMarkets" or "ExcludedMarkets" + or "UnsupportedOSArchitectures" or "UnsupportedArguments" or "ElevationRequirement" + or "RepairBehavior" or "InstallerIdentifier" or "ProductId" or "StoreProductId"; + + private static JsonNode ConvertScalar(YamlScalarNode scalar) + { + var value = scalar.Value ?? string.Empty; + if (scalar.Style == YamlDotNet.Core.ScalarStyle.Plain + && (value is "~" || value.Equals("null", StringComparison.OrdinalIgnoreCase))) + return null!; + if (scalar.Style == YamlDotNet.Core.ScalarStyle.Plain + && (value.Equals("true", StringComparison.OrdinalIgnoreCase) || value.Equals("false", StringComparison.OrdinalIgnoreCase))) + return JsonValue.Create(value.Equals("true", StringComparison.OrdinalIgnoreCase))!; + if (scalar.Style == YamlDotNet.Core.ScalarStyle.Plain + && long.TryParse(value, System.Globalization.NumberStyles.AllowLeadingSign, System.Globalization.CultureInfo.InvariantCulture, out var integer) + && !value.Contains('.')) + return JsonValue.Create(integer)!; + if (scalar.Style == YamlDotNet.Core.ScalarStyle.Plain + && ulong.TryParse(value, System.Globalization.NumberStyles.AllowLeadingSign, System.Globalization.CultureInfo.InvariantCulture, out var unsignedInteger)) + return JsonValue.Create(unsignedInteger)!; + if (scalar.Style == YamlDotNet.Core.ScalarStyle.Plain + && (value.Contains('.') || value.Contains('e') || value.Contains('E')) + && double.TryParse(value, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var floating)) + { + if (!double.IsFinite(floating)) + throw new PublisherException("Non-finite YAML numbers cannot be preserved in JSON-compatible manifests."); + return JsonNode.Parse(FormatFloatingPoint(floating))!; + } + return JsonValue.Create(value)!; + } + + private static string FormatFloatingPoint(double value) + { + var text = value.ToString("R", System.Globalization.CultureInfo.InvariantCulture).ToLowerInvariant(); + var sign = text.StartsWith('-') ? "-" : string.Empty; + text = text.TrimStart('-'); + var exponentOffset = text.IndexOf('e'); + var exponent = exponentOffset < 0 ? 0 : int.Parse(text[(exponentOffset + 1)..], System.Globalization.CultureInfo.InvariantCulture); + var mantissa = exponentOffset < 0 ? text : text[..exponentOffset]; + var point = mantissa.IndexOf('.'); + if (point >= 0) + exponent -= mantissa.Length - point - 1; + var digits = mantissa.Replace(".", string.Empty, StringComparison.Ordinal).TrimStart('0'); + if (digits.Length == 0) + return sign + "0.0"; + while (digits.Length > 1 && digits.EndsWith('0')) + { + digits = digits[..^1]; + exponent++; + } + var decimalPosition = digits.Length + exponent; + if (decimalPosition > 0 && decimalPosition <= 16) + return sign + (decimalPosition >= digits.Length + ? digits + new string('0', decimalPosition - digits.Length) + ".0" + : digits.Insert(decimalPosition, ".")); + if (decimalPosition <= 0 && decimalPosition > -5) + return sign + "0." + new string('0', -decimalPosition) + digits; + var scientificExponent = decimalPosition - 1; + return sign + digits[0] + (digits.Length > 1 ? "." + digits[1..] : string.Empty) + + "e" + (scientificExponent >= 0 ? "+" : string.Empty) + + scientificExponent.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + + private static List MergeDocuments( + Dictionary<(string Id, string Version), List> documentGroups) + { + var result = new List(); + foreach (var entry in documentGroups.OrderBy(x => x.Key.Id, StringComparer.OrdinalIgnoreCase) + .ThenBy(x => x.Key.Version, StringComparer.Ordinal)) + { + var docs = entry.Value; + var singleton = docs.Where(d => d.Type == "singleton").ToList(); + if (singleton.Count > 1) + throw new PublisherException($"Duplicate singleton manifest for {entry.Key.Id} {entry.Key.Version}."); + if (singleton.Count != 0) + { + if (docs.Count != 1) + throw new PublisherException($"Singleton and multi-file manifests conflict for {entry.Key.Id} {entry.Key.Version}."); + var singletonModel = (JsonObject)singleton[0].Model.DeepClone(); + var locale = OptionalString(singletonModel, "PackageLocale") + ?? throw new PublisherException($"Singleton manifest is missing PackageLocale: {singleton[0].Path}"); + if (!LocalePattern().IsMatch(locale)) + throw new PublisherException($"Invalid PackageLocale '{locale}' in {singleton[0].Path}."); + AddMerged(result, entry.Key.Id, entry.Key.Version, locale, locale, singletonModel); + continue; + } + + var versionDocs = docs.Where(d => d.Type == "version").ToList(); + var installerDocs = docs.Where(d => d.Type == "installer").ToList(); + var defaultDocs = docs.Where(d => d.Type == "defaultLocale").ToList(); + var localeDocs = docs.Where(d => d.Type == "locale").ToList(); + if (versionDocs.Count != 1 || defaultDocs.Count != 1) + throw new PublisherException($"Multi-file set {entry.Key.Id} {entry.Key.Version} requires exactly one version and defaultLocale manifest; missing or duplicate documents are not allowed."); + if (installerDocs.Count != 1 || installerDocs[0].Model["Installers"] is not JsonArray { Count: > 0 }) + throw new PublisherException($"Multi-file set {entry.Key.Id} {entry.Key.Version} requires exactly one installer manifest with nonempty Installers."); + var defaultLocale = defaultDocs[0].Locale!; + var declaredDefaultLocale = RequiredString(versionDocs[0].Model, "DefaultLocale", versionDocs[0].Path); + if (!string.Equals(declaredDefaultLocale, defaultLocale, StringComparison.OrdinalIgnoreCase)) + throw new PublisherException($"DefaultLocale '{declaredDefaultLocale}' does not match defaultLocale document '{defaultLocale}'."); + var localeGroups = localeDocs.GroupBy(d => d.Locale!, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase); + foreach (var group in localeGroups) + if (group.Value.Count != 1) + throw new PublisherException($"Duplicate locale manifest '{group.Key}' for {entry.Key.Id} {entry.Key.Version}."); + if (localeGroups.ContainsKey(defaultLocale)) + throw new PublisherException($"A locale manifest duplicates the default locale '{defaultLocale}' for {entry.Key.Id} {entry.Key.Version}."); + var model = new JsonObject(); + MergeObject(model, versionDocs[0].Model); + MergeObject(model, installerDocs[0].Model); + MergeObject(model, defaultDocs[0].Model); + var localizations = model["Localization"] as JsonArray ?? new JsonArray(); + if (model["Localization"] is null) + model["Localization"] = localizations; + foreach (var localeGroup in localeGroups.OrderBy(x => x.Key, StringComparer.Ordinal)) + { + var localization = (JsonObject)localeGroup.Value[0].Model.DeepClone(); + foreach (var key in new[] { "ManifestType", "ManifestVersion", "PackageIdentifier", "PackageVersion" }) + localization.Remove(key); + localizations.Add((JsonNode)localization); + } + if (localizations.Count == 0) + model.Remove("Localization"); + model["ManifestType"] = "singleton"; + AddMerged(result, entry.Key.Id, entry.Key.Version, defaultLocale, defaultLocale, model); + } + return result; + } + + private static void MergeObject(JsonObject target, JsonObject source) + { + foreach (var pair in source) + { + if (pair.Key is "ManifestType" or "PackageIdentifier" or "PackageVersion" or "ManifestVersion" + && target.ContainsKey(pair.Key)) + continue; + if (pair.Value is JsonObject child && target[pair.Key] is JsonObject existing) + MergeObject(existing, child); + else + target[pair.Key] = pair.Value?.DeepClone(); + } + } + + private static void AddMerged(List target, string id, string version, string locale, string defaultLocale, JsonObject model) + { + ValidatePathSegment(locale, "PackageLocale"); + _ = RequiredString(model, "PackageName", id); + _ = RequiredString(model, "Publisher", id); + if (model["Installers"] is not JsonArray { Count: > 0 }) + throw new PublisherException($"Manifest {id} {version} requires nonempty Installers."); + var content = CanonicalJson(model); + var hash = Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(content)).ToLowerInvariant(); + var relativePath = $"manifests/{id}/{version}/{locale}/{hash[..16]}.yaml"; + target.Add(new MergedManifest(id, version, locale, defaultLocale, relativePath, content, hash, model)); + } + + private static ScanResult BuildPackages(SortedDictionary inventory, List manifests) + { + var packages = new List(); + foreach (var group in manifests.GroupBy(m => m.PackageId, StringComparer.OrdinalIgnoreCase)) + { + var versions = ValidateVersions(group); + var latestVersion = versions[^1]; + var latest = group.First(m => m.Version.Equals(latestVersion, StringComparison.OrdinalIgnoreCase) + && m.Locale.Equals(m.DefaultLocale, StringComparison.OrdinalIgnoreCase)); + var all = group.ToArray(); + var sidecarYaml = BuildVersionData(all); + var compressed = MsZipCompressor.Compress(sidecarYaml); + var tags = Values(all, "Tags"); + var commands = Values(all, "Commands"); + var pfns = Values(all, "PackageFamilyName"); + var products = Values(all, "ProductCode"); + var upgradeCodes = Values(all, "UpgradeCode"); + var normalizedNames = new HashSet(StringComparer.Ordinal); + var normalizedPublishers = new HashSet(StringComparer.Ordinal); + foreach (var manifest in all) + { + foreach (var pair in AppEntries(manifest.Model)) + { + if (pair.DisplayName is not null) + normalizedNames.Add(NormalizeName(pair.DisplayName)); + if (pair.Publisher is not null) + normalizedPublishers.Add(NormalizePublisher(pair.Publisher)); + if (pair.ProductCode is not null) + products = products.Append(pair.ProductCode).ToArray(); + if (pair.UpgradeCode is not null) + upgradeCodes = upgradeCodes.Append(pair.UpgradeCode).ToArray(); + } + foreach (var scope in CatalogScopes(manifest.Model)) + { + var name = OptionalString(scope, "PackageName"); + var publisher = OptionalString(scope, "Publisher"); + if (name is not null) + normalizedNames.Add(NormalizeName(name)); + if (publisher is not null) + normalizedPublishers.Add(NormalizePublisher(publisher)); + } + } + + var arpVersions = all.SelectMany(m => AppEntries(m.Model)) + .SelectMany(entry => new[] { entry.DisplayVersion }) + .Where(v => !string.IsNullOrWhiteSpace(v)) + .Cast() + .OrderBy(v => v, VersionComparer.Instance) + .ToArray(); + packages.Add(new IndexedPackage( + group.Key, + RequiredString(latest.Model, "PackageName", latest.RelativePath), + OptionalString(latest.Model, "Moniker"), + latestVersion, + all, + tags, + commands, + pfns, + DistinctTerms(products), + DistinctTerms(upgradeCodes), + DistinctTerms(normalizedNames), + DistinctTerms(normalizedPublishers), + arpVersions.FirstOrDefault(), + arpVersions.LastOrDefault(), + sidecarYaml, + compressed, + System.Security.Cryptography.SHA256.HashData(compressed))); + } + + return new ScanResult(inventory, packages, manifests); + } + + private static string[] ValidateVersions(IEnumerable group) + { + if (group.GroupBy(m => m.Version, StringComparer.OrdinalIgnoreCase) + .Any(g => g.Select(m => m.Version).Distinct(StringComparer.Ordinal).Count() > 1)) + throw new PublisherException("Package version differs only by casing."); + var versions = group.Select(m => m.Version).OrderBy(v => v, VersionComparer.Instance).ToArray(); + for (var index = 1; index < versions.Length; index++) + if (VersionComparer.Instance.Compare(versions[index - 1], versions[index]) == 0) + throw new PublisherException($"Package versions '{versions[index - 1]}' and '{versions[index]}' compare equally; refusing ambiguous latest-version selection."); + return versions; + } + + private static byte[] BuildVersionData(IReadOnlyList manifests) + { + var sorted = manifests.OrderByDescending(m => m.Version, VersionComparer.Instance) + .ThenBy(m => m.Locale, StringComparer.Ordinal) + .ToArray(); + var builder = new StringBuilder("sV: \"1.0\"\nvD:\n"); + foreach (var manifest in sorted) + { + builder.Append(" - v: ").Append(QuoteYaml(manifest.Version)).Append('\n'); + var arpVersions = AppEntries(manifest.Model) + .Select(entry => entry.DisplayVersion) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Cast() + .OrderBy(value => value, VersionComparer.Instance) + .ToArray(); + if (arpVersions.Length > 0) + { + builder.Append(" aMiV: ").Append(QuoteYaml(arpVersions[0])).Append('\n'); + builder.Append(" aMaV: ").Append(QuoteYaml(arpVersions[^1])).Append('\n'); + } + builder.Append(" rP: ").Append(QuoteYaml(manifest.RelativePath)).Append('\n'); + builder.Append(" s256H: ").Append(QuoteYaml(manifest.Hash)).Append('\n'); + } + return Encoding.UTF8.GetBytes(builder.ToString()); + } + + private static IReadOnlyList Values(IReadOnlyList manifests, string key) => + DistinctTerms(manifests.SelectMany(m => CatalogScopes(m.Model).SelectMany(obj => StringValues(obj[key])))); + + private static IReadOnlyList DistinctTerms(IEnumerable terms) => + terms.Where(s => !string.IsNullOrWhiteSpace(s)).Order(StringComparer.Ordinal) + .GroupBy(s => new string(s.Select(c => c is >= 'A' and <= 'Z' ? (char)(c + 32) : c).ToArray()), StringComparer.Ordinal) + .Select(g => g.First()).Order(StringComparer.Ordinal).ToArray(); + + private static IEnumerable CatalogScopes(JsonObject model) + { + yield return model; + foreach (var key in new[] { "Installers", "Localization" }) + if (model[key] is JsonArray array) + foreach (var item in array.OfType()) + yield return item; + } + + private static IEnumerable StringValues(JsonNode? value) + { + if (value is JsonArray array) + { + foreach (var item in array) + if (item is JsonValue json && json.TryGetValue(out var text) && text is not null) + yield return text; + } + else if (value is JsonValue scalar && scalar.TryGetValue(out var text) && text is not null) + { + yield return text; + } + } + + private static IEnumerable<(string? DisplayName, string? Publisher, string? ProductCode, string? UpgradeCode, string? DisplayVersion)> AppEntries(JsonObject model) + { + foreach (var obj in CatalogScopes(model)) + { + if (obj["AppsAndFeaturesEntries"] is JsonArray entries) + foreach (var entry in entries.OfType()) + yield return ( + OptionalString(entry, "DisplayName"), + OptionalString(entry, "Publisher"), + OptionalString(entry, "ProductCode"), + OptionalString(entry, "UpgradeCode"), + OptionalString(entry, "DisplayVersion")); + } + } + + private static IEnumerable Descendants(JsonObject model) + { + yield return model; + foreach (var child in model.Select(pair => pair.Value)) + { + if (child is JsonObject nested) + foreach (var item in Descendants(nested)) + yield return item; + else if (child is JsonArray array) + foreach (var nestedObject in array.OfType()) + foreach (var item in Descendants(nestedObject)) + yield return item; + } + } + + private static string NormalizeName(string value) + { + var normalized = Devolutions.Pinget.Core.NameNormalization.NormalizeName(value).Name; + return normalized; + } + + private static string NormalizePublisher(string value) => + Devolutions.Pinget.Core.NameNormalization.NormalizePublisher(value); + + private static string QuoteYaml(string value) => + $"\"{JsonEncodedText.Encode(value, JavaScriptEncoder.UnsafeRelaxedJsonEscaping)}\""; + + private static byte[] CanonicalJson(JsonObject model) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping })) + WriteCanonical(writer, model); + stream.WriteByte((byte)'\n'); + return stream.ToArray(); + } + + private static void WriteCanonical(Utf8JsonWriter writer, JsonNode? node) + { + switch (node) + { + case JsonObject obj: + writer.WriteStartObject(); + foreach (var pair in obj.OrderBy(p => p.Key, StringComparer.Ordinal)) + { + writer.WritePropertyName(pair.Key); + WriteCanonical(writer, pair.Value); + } + writer.WriteEndObject(); + break; + case JsonArray array: + writer.WriteStartArray(); + foreach (var item in array) + WriteCanonical(writer, item); + writer.WriteEndArray(); + break; + default: + if (node is null) + writer.WriteNullValue(); + else + node.WriteTo(writer); + break; + } + } + + private static void RejectOutputCollisions(IEnumerable manifests) + { + var paths = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var manifest in manifests) + if (!paths.Add(manifest.RelativePath)) + throw new PublisherException($"Manifest output path collision: {manifest.RelativePath}"); + } + + private static void ValidatePathSegment(string value, string label) + { + if (value is "." or ".." || value.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 + || value.Contains('/') || value.Contains('\\') || value.Contains(':')) + throw new PublisherException($"Unsafe {label} path component: '{value}'."); + } + + private static string RequiredString(JsonObject model, string key, string path) + { + var value = OptionalString(model, key); + return !string.IsNullOrWhiteSpace(value) + ? value + : throw new PublisherException($"Required field '{key}' is missing in {path}."); + } + + private static string? OptionalString(JsonObject model, string key) => + model[key] is JsonValue value && value.TryGetValue(out var text) ? text : null; + + private sealed record ManifestDocument(string Type, string? Locale, string Path, JsonObject Model); + + private sealed class VersionComparer : IComparer + { + public static VersionComparer Instance { get; } = new(); + + public int Compare(string? x, string? y) + { + var left = ParsedVersion.Parse(x ?? string.Empty); + var right = ParsedVersion.Parse(y ?? string.Empty); + if (left.IsLatest != right.IsLatest) + return left.IsLatest ? 1 : -1; + if (left.IsUnknown != right.IsUnknown) + return left.IsUnknown ? -1 : 1; + for (var index = 0; index < Math.Max(left.Parts.Count, right.Parts.Count); index++) + { + var part = ComparePart( + index < left.Parts.Count ? left.Parts[index] : new VersionPart(0, string.Empty), + index < right.Parts.Count ? right.Parts[index] : new VersionPart(0, string.Empty)); + if (part != 0) + return part; + } + return left.Approximation.CompareTo(right.Approximation); + } + + private static int ComparePart(VersionPart left, VersionPart right) + { + var integer = left.Integer.CompareTo(right.Integer); + if (integer != 0) + return integer; + if (left.Other.Length == 0) + return right.Other.Length == 0 ? 0 : 1; + if (right.Other.Length == 0) + return -1; + return StringComparer.Ordinal.Compare(left.Other, right.Other); + } + + private sealed record ParsedVersion(bool IsLatest, bool IsUnknown, int Approximation, IReadOnlyList Parts) + { + public static ParsedVersion Parse(string source) + { + var value = source.Trim(); + var approximation = 0; + if (value.StartsWith("< ", StringComparison.OrdinalIgnoreCase)) + { + approximation = -1; + value = value[2..]; + } + else if (value.StartsWith("> ", StringComparison.OrdinalIgnoreCase)) + { + approximation = 1; + value = value[2..]; + } + var firstDigit = value.IndexOfAny("0123456789".ToCharArray()); + var firstSeparator = value.IndexOf('.'); + if (firstDigit >= 0 && (firstSeparator < 0 || firstDigit < firstSeparator)) + value = value[firstDigit..]; + var parts = value.Split('.').Select(VersionPart.Parse).ToList(); + while (parts.Count > 0 && parts[^1].Integer == 0 && parts[^1].Other.Length == 0) + parts.RemoveAt(parts.Count - 1); + var latest = parts.Count == 1 && parts[0].Integer == 0 && parts[0].Other.Equals("latest", StringComparison.OrdinalIgnoreCase); + var unknown = parts.Count == 1 && parts[0].Integer == 0 && parts[0].Other.Equals("unknown", StringComparison.OrdinalIgnoreCase); + return new ParsedVersion(latest, unknown, approximation, parts); + } + } + + private readonly record struct VersionPart(ulong Integer, string Other) + { + public static VersionPart Parse(string raw) + { + var part = raw.Trim(); + var digits = 0; + while (digits < part.Length && char.IsAsciiDigit(part[digits])) + digits++; + if (digits == 0 || !ulong.TryParse(part.AsSpan(0, digits), System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out var integer)) + return new VersionPart(0, part.ToLowerInvariant()); + return new VersionPart(integer, part[digits..].ToLowerInvariant()); + } + } + } + + [GeneratedRegex(@"^[A-Za-z0-9][A-Za-z0-9.-]{0,127}$", RegexOptions.CultureInvariant)] + private static partial Regex IdentifierPattern(); + + [GeneratedRegex(@"^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$", RegexOptions.CultureInvariant)] + private static partial Regex VersionPattern(); + + [GeneratedRegex(@"^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$", RegexOptions.CultureInvariant)] + private static partial Regex LocalePattern(); +} diff --git a/dotnet/src/Devolutions.Pinget.Publisher/MsZipCompressor.cs b/dotnet/src/Devolutions.Pinget.Publisher/MsZipCompressor.cs new file mode 100644 index 0000000..1c50105 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/MsZipCompressor.cs @@ -0,0 +1,111 @@ +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Pinget.Publisher; + +internal static class MsZipCompressor +{ + private const uint CompressAlgorithmMsZip = 2; + private const int ErrorInsufficientBuffer = 122; + private const int MaximumSize = 64 * 1024 * 1024; + + public static byte[] Compress(byte[] input) + { + if (input.Length > MaximumSize) + throw new PublisherException("MSZIP input exceeds the 64 MiB limit."); + if (!OperatingSystem.IsWindows()) + throw new PublisherException("WinGet v2 MSZIP generation currently requires Windows Compression API support."); + if (!CreateCompressor(CompressAlgorithmMsZip, IntPtr.Zero, out var compressor)) + throw new PublisherException("Windows Compression API could not create an MSZIP compressor.", LastError()); + try + { + _ = CompressBuffer(compressor, input, (nuint)input.Length, null, 0, out var required); + var error = Marshal.GetLastWin32Error(); + if (error != ErrorInsufficientBuffer || required == 0) + throw new PublisherException("Windows Compression API did not report the required MSZIP output buffer size.", new Win32Exception(error)); + + var output = new byte[checked((int)required)]; + if (!CompressBuffer(compressor, input, (nuint)input.Length, output, (nuint)output.Length, out var written)) + throw new PublisherException("Windows Compression API failed to create the MSZIP stream.", LastError()); + Array.Resize(ref output, checked((int)written)); + return output; + } + finally + { + _ = CloseCompressor(compressor); + } + } + + public static bool IsAvailable() + { + if (!OperatingSystem.IsWindows()) + return false; + if (!CreateCompressor(CompressAlgorithmMsZip, IntPtr.Zero, out var compressor)) + return false; + return CloseCompressor(compressor); + } + + public static byte[] Decompress(byte[] input) + { + if (!OperatingSystem.IsWindows()) + throw new PublisherException("WinGet v2 MSZIP verification currently requires Windows Compression API support."); + if (!CreateDecompressor(CompressAlgorithmMsZip, IntPtr.Zero, out var decompressor)) + throw new PublisherException("Windows Compression API could not create an MSZIP decompressor.", LastError()); + try + { + _ = DecompressBuffer(decompressor, input, (nuint)input.Length, null, 0, out var required); + var error = Marshal.GetLastWin32Error(); + if (error != ErrorInsufficientBuffer) + throw new PublisherException("Windows Compression API did not report the required decompressed buffer size.", new Win32Exception(error)); + if (required > MaximumSize) + throw new PublisherException("MSZIP output exceeds the 64 MiB decompression limit."); + var output = new byte[checked((int)required)]; + if (!DecompressBuffer(decompressor, input, (nuint)input.Length, output, (nuint)output.Length, out var written)) + throw new PublisherException("Windows Compression API failed to decode the MSZIP stream.", LastError()); + Array.Resize(ref output, checked((int)written)); + return output; + } + finally + { + _ = CloseDecompressor(decompressor); + } + } + + private static Exception LastError() => new Win32Exception(Marshal.GetLastWin32Error()); + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CreateCompressor(uint algorithm, IntPtr options, out IntPtr compressor); + + [DllImport("cabinet.dll", SetLastError = true, EntryPoint = "Compress")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CompressBuffer( + IntPtr compressor, + byte[] input, + nuint inputSize, + byte[]? output, + nuint outputSize, + out nuint outputWritten); + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CloseCompressor(IntPtr compressor); + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CreateDecompressor(uint algorithm, IntPtr options, out IntPtr decompressor); + + [DllImport("cabinet.dll", SetLastError = true, EntryPoint = "Decompress")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool DecompressBuffer( + IntPtr decompressor, + byte[] input, + nuint inputSize, + byte[]? output, + nuint outputSize, + out nuint outputWritten); + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CloseDecompressor(IntPtr decompressor); +} diff --git a/dotnet/src/Devolutions.Pinget.Publisher/PublisherModels.cs b/dotnet/src/Devolutions.Pinget.Publisher/PublisherModels.cs new file mode 100644 index 0000000..5fe34dd --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/PublisherModels.cs @@ -0,0 +1,53 @@ +namespace Pinget.Publisher; + +public sealed record PublisherBuildOptions(string InputDirectory, string StateDirectory); + +public sealed record PublisherBuildResult( + string BuildId, + string SnapshotDirectory, + int ManifestCount, + int PackageCount, + int ChangedFiles, + int RemovedFiles); + +public sealed record PublisherDiffResult( + string? LastBuildId, + bool Published, + IReadOnlyList Added, + IReadOnlyList Changed, + IReadOnlyList Removed); + +public sealed record PublisherStatusResult( + string? LastBuildId, + string? LastPublishedBuildId, + int BuildCount, + string? LastBuildTimestamp); + +public sealed record PublisherDoctorResult( + string OperatingSystem, + string? MakeAppxPath, + string? SignToolPath, + bool NativeMsZipAvailable); + +public sealed record PublisherPublishOptions( + string StateDirectory, + string DestinationDirectory, + string PackageName, + string Publisher, + string PackageVersion, + string? PfxPath = null, + string? PasswordEnvironmentVariable = null, + bool UnsignedDevelopment = false, + string? MakeAppxPath = null, + string? SignToolPath = null, + string? BuildId = null, + string? TimestampUrl = null); + +public sealed record PublisherVerifyResult( + bool Valid, + string Subject, + int ArtifactCount, + string? BuildId); + +public sealed class PublisherException(string message, Exception? inner = null) + : Exception(message, inner); diff --git a/dotnet/src/Devolutions.Pinget.Publisher/PublisherPackaging.cs b/dotnet/src/Devolutions.Pinget.Publisher/PublisherPackaging.cs new file mode 100644 index 0000000..6606aa3 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/PublisherPackaging.cs @@ -0,0 +1,563 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Xml.Linq; + +namespace Pinget.Publisher; + +public static partial class PublisherPackaging +{ + private const string MarkerName = ".pinget-publisher.json"; + private const string PackageExtensionId = "IndexDB"; + private static readonly byte[] LogoPng = Convert.FromHexString( + "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000b49444154789c636000020000050001a5f645400000000049454e44ae426082"); + + public static PublisherDoctorResult Doctor(string? makeAppxPath = null, string? signToolPath = null) + { + var makeAppx = FindTool(makeAppxPath, "makeappx.exe"); + var signTool = FindTool(signToolPath, "signtool.exe"); + return new PublisherDoctorResult( + System.Runtime.InteropServices.RuntimeInformation.OSDescription, + makeAppx, + signTool, + MsZipCompressor.IsAvailable()); + } + + public static PublisherVerifyResult VerifyDestination( + string destinationDirectory, + string? signToolPath = null) + { + var destination = Path.GetFullPath(destinationDirectory); + if (!Directory.Exists(destination)) + throw new PublisherException($"Published source directory does not exist: {destination}"); + PublisherWorkspace.EnsureSafeExistingPath(destination); + var markerPath = Path.Combine(destination, MarkerName); + if (!File.Exists(markerPath)) + throw new PublisherException($"Destination is not owned by Pinget publisher; missing {MarkerName}."); + var marker = ReadMarker(markerPath); + if (marker["schemaVersion"]?.GetValue() != 1 || marker["owner"]?.GetValue() != "Pinget.Publisher") + throw new PublisherException("Unsupported publisher ownership marker."); + var inventory = marker["artifacts"] as JsonObject + ?? throw new PublisherException("Publisher ownership marker has no artifact inventory."); + var directories = new Stack(); + directories.Push(destination); + while (directories.TryPop(out var directory)) + { + foreach (var entry in Directory.EnumerateFileSystemEntries(directory)) + { + var attributes = File.GetAttributes(entry); + if ((attributes & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Reparse point is not allowed in published source: {entry}"); + if ((attributes & FileAttributes.Directory) != 0) + { + directories.Push(entry); + continue; + } + var relativePath = Path.GetRelativePath(destination, entry).Replace('\\', '/'); + if (relativePath != MarkerName && !inventory.ContainsKey(relativePath)) + throw new PublisherException($"Published source contains an untracked artifact: {relativePath}"); + } + } + foreach (var item in inventory) + { + var path = PublisherWorkspace.ResolveContained(destination, item.Key); + PublisherWorkspace.EnsureSafeExistingPath(Path.GetDirectoryName(path)!); + if (File.Exists(path) && (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Reparse point is not allowed for published artifact: {item.Key}"); + if (!File.Exists(path)) + throw new PublisherException($"Published artifact is missing: {item.Key}"); + var actual = HashFile(path); + if (!string.Equals(item.Value?.GetValue(), actual, StringComparison.Ordinal)) + throw new PublisherException($"Published artifact hash mismatch: {item.Key}"); + } + + var packagePath = Path.Combine(destination, "source2.msix"); + if (!File.Exists(packagePath)) + throw new PublisherException("Published source2.msix is missing."); + VerifyMsixContents(packagePath, marker); + var unsigned = marker["unsignedDevelopment"]?.GetValue() ?? false; + if (!unsigned) + { + if (!OperatingSystem.IsWindows()) + throw new PublisherException("MSIX signature verification requires Windows signtool on this platform."); + var signtool = FindTool(signToolPath, "signtool.exe") + ?? throw new PublisherException("signtool.exe was not found. Use --signtool to specify it."); + RunTool(signtool, ["verify", "/pa", "/v", packagePath], "MSIX signature verification"); + } + return new PublisherVerifyResult(true, "published WinGet v2 static source", inventory.Count, marker["buildId"]?.GetValue()); + } + + public static PublisherVerifyResult Publish(PublisherPublishOptions options) + { + if (!OperatingSystem.IsWindows()) + throw new PublisherException("MSIX packaging and signing are supported only on Windows in this build."); + var state = Path.GetFullPath(options.StateDirectory); + var destination = Path.TrimEndingDirectorySeparator(Path.GetFullPath(options.DestinationDirectory)); + PublisherWorkspace.EnsureDisjoint(state, destination, "state", "destination"); + PublisherWorkspace.EnsureSafeExistingPath(state); + if (!Directory.Exists(state)) + throw new PublisherException($"Publisher state does not exist: {state}"); + ValidateIdentity(options.PackageName, options.Publisher); + var version = ParsePackageVersion(options.PackageVersion); + if (options.UnsignedDevelopment == (options.PfxPath is not null)) + throw new PublisherException("Choose exactly one: --unsigned-development or --pfx with a signing password environment variable."); + if (options.PfxPath is not null && string.IsNullOrWhiteSpace(options.PasswordEnvironmentVariable)) + throw new PublisherException("Signing requires --password-env naming the environment variable containing the PFX password."); + if (options.UnsignedDevelopment && options.PasswordEnvironmentVariable is not null) + throw new PublisherException("--unsigned-development cannot be combined with --password-env."); + if (options.TimestampUrl is not null + && (!Uri.TryCreate(options.TimestampUrl, UriKind.Absolute, out var timestampUri) || timestampUri.Scheme != Uri.UriSchemeHttps)) + throw new PublisherException("--timestamp-url must be an absolute HTTPS URL."); + using (var existingState = PublisherWorkspace.OpenState(state, create: false)) + { + } + using var writerLock = PublisherWorkspace.AcquireLock(state); + var destinationParent = Path.GetDirectoryName(destination) + ?? throw new PublisherException("Publication destination must not be a filesystem root."); + PublisherWorkspace.EnsureSafeExistingPath(destinationParent); + Directory.CreateDirectory(destinationParent); + var destinationLockPath = Path.Combine(destinationParent, Path.GetFileName(destination) + ".pinget-publisher.lock"); + if (File.Exists(destinationLockPath) && (File.GetAttributes(destinationLockPath) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException("Destination lock cannot be a reparse point."); + using var destinationLock = new FileStream(destinationLockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + + var doctor = Doctor(options.MakeAppxPath, options.SignToolPath); + var makeappx = doctor.MakeAppxPath ?? throw new PublisherException("makeappx.exe was not found. Use --makeappx to specify it."); + var signTool = options.UnsignedDevelopment + ? null + : doctor.SignToolPath ?? throw new PublisherException("signtool.exe was not found. Use --signtool to specify it."); + + var workspace = new PublisherWorkspace(state); + var buildId = options.BuildId ?? workspace.Status().LastBuildId + ?? throw new PublisherException("Publisher state has no successful build to publish."); + _ = workspace.Verify(buildId); + + PublisherWorkspace.EnsureSafeExistingPath(destination); + Directory.CreateDirectory(destination); + PublisherWorkspace.EnsureSafeExistingPath(destination); + var markerPath = Path.Combine(destination, MarkerName); + var marker = ExistingDestinationMarker(destination, markerPath); + if (marker is not null) + _ = VerifyDestination(destination, signTool); + var oldBuildId = marker?["buildId"]?.GetValue(); + var oldVersionText = marker?["packageVersion"]?.GetValue(); + if (marker is not null) + { + var existingIdentity = marker["packageName"]?.GetValue(); + var existingPublisher = marker["publisher"]?.GetValue(); + if (!string.Equals(existingIdentity, options.PackageName, StringComparison.Ordinal) + || !string.Equals(existingPublisher, options.Publisher, StringComparison.Ordinal)) + throw new PublisherException("The destination is already owned by a different source package identity or publisher."); + if (oldVersionText is null || !Version.TryParse(oldVersionText, out var oldVersion)) + throw new PublisherException("Existing publisher marker has an invalid source package version."); + var comparison = version.CompareTo(oldVersion); + if (comparison < 0) + throw new PublisherException($"Source package version {version} must be greater than published version {oldVersion}."); + if (comparison == 0) + { + if (!string.Equals(oldBuildId, buildId, StringComparison.Ordinal) + || (marker["unsignedDevelopment"]?.GetValue() ?? false) != options.UnsignedDevelopment) + throw new PublisherException("The source package version has already been used for different build content or signing mode."); + var existing = VerifyDestination(destination, signTool); + PublisherWorkspace.RecordPublished(state, buildId); + return existing; + } + } + + var buildPath = Path.Combine(state, "builds", buildId); + var stagingRoot = Path.Combine(state, $".publish-{Guid.NewGuid():N}"); + var packageStage = Path.Combine(stagingRoot, "package"); + Directory.CreateDirectory(packageStage); + var createdArtifacts = new List(); + var createdDirectories = new List(); + var promoted = false; + try + { + File.Copy(Path.Combine(buildPath, "Public", "index.db"), Path.Combine(Directory.CreateDirectory(Path.Combine(packageStage, "Public")).FullName, "index.db")); + var assets = Directory.CreateDirectory(Path.Combine(packageStage, "Assets")); + foreach (var asset in new[] { "StoreLogo.png", "Logo150.png", "Logo44.png" }) + File.WriteAllBytes(Path.Combine(assets.FullName, asset), LogoPng); + WriteAppxManifest(Path.Combine(packageStage, "AppxManifest.xml"), options, version); + + var stagedPackage = Path.Combine(stagingRoot, "source2.msix"); + RunTool(makeappx, ["pack", "/d", packageStage, "/p", stagedPackage, "/o"], "MSIX packaging"); + if (!File.Exists(stagedPackage)) + throw new PublisherException("makeappx completed without producing source2.msix."); + if (!options.UnsignedDevelopment) + { + var password = Environment.GetEnvironmentVariable(options.PasswordEnvironmentVariable!); + if (string.IsNullOrEmpty(password)) + throw new PublisherException($"Signing password environment variable '{options.PasswordEnvironmentVariable}' is not set or empty."); + ValidateCertificate(options.PfxPath!, password, options.Publisher); + var signingArguments = new List { "sign", "/fd", "SHA256", "/f", Path.GetFullPath(options.PfxPath!), "/p", password }; + if (options.TimestampUrl is not null) + signingArguments.AddRange(["/tr", options.TimestampUrl, "/td", "SHA256"]); + signingArguments.Add(stagedPackage); + RunTool(signTool!, signingArguments, "MSIX signing", password); + RunTool(signTool!, ["verify", "/pa", "/v", stagedPackage], "MSIX signature verification"); + } + + var existingArtifacts = marker?["artifacts"]?.DeepClone() as JsonObject ?? new JsonObject(); + var snapshotArtifacts = ReadSnapshotArtifacts(buildPath); + foreach (var item in snapshotArtifacts) + { + var artifactHash = item.Value?.GetValue() + ?? throw new PublisherException($"Invalid artifact hash for '{item.Key}'."); + var source = PublisherWorkspace.ResolveContained(Path.Combine(buildPath, "feed"), item.Key); + var target = PublisherWorkspace.ResolveContained(destination, item.Key); + PublisherWorkspace.EnsureSafeExistingPath(Path.GetDirectoryName(target)!); + var targetDirectory = Path.GetDirectoryName(target)!; + var missingDirectories = new Stack(); + for (var directory = targetDirectory; !Directory.Exists(directory); directory = Path.GetDirectoryName(directory)!) + missingDirectories.Push(directory); + while (missingDirectories.TryPop(out var missingDirectory)) + { + Directory.CreateDirectory(missingDirectory); + createdDirectories.Add(missingDirectory); + } + if (File.Exists(target)) + { + if ((File.GetAttributes(target) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Reparse point is not allowed for published artifact: {item.Key}"); + if (!string.Equals(HashFile(target), artifactHash, StringComparison.Ordinal)) + throw new PublisherException($"Refusing to overwrite immutable published artifact with different bytes: {item.Key}"); + } + else + { + CopyAtomically(source, target, overwrite: false); + createdArtifacts.Add(target); + if (!string.Equals(HashFile(target), artifactHash, StringComparison.Ordinal)) + throw new PublisherException($"Copied artifact hash mismatch: {item.Key}"); + } + existingArtifacts[item.Key] = artifactHash; + } + + var packageHash = HashFile(stagedPackage); + existingArtifacts["source2.msix"] = packageHash; + var updatedMarker = new JsonObject + { + ["schemaVersion"] = 1, + ["owner"] = "Pinget.Publisher", + ["packageName"] = options.PackageName, + ["publisher"] = options.Publisher, + ["packageVersion"] = version.ToString(), + ["buildId"] = buildId, + ["indexHash"] = ReadSnapshotIndexHash(buildPath), + ["unsignedDevelopment"] = options.UnsignedDevelopment, + ["publishedUtc"] = DateTimeOffset.UtcNow.ToString("O"), + ["artifacts"] = existingArtifacts, + }; + var oldMarkerBytes = File.Exists(markerPath) ? File.ReadAllBytes(markerPath) : null; + VerifyMsixContents(stagedPackage, updatedMarker); + WriteJsonAtomically(markerPath, updatedMarker); + try + { + CopyAtomically(stagedPackage, Path.Combine(destination, "source2.msix")); + } + catch + { + if (oldMarkerBytes is null) + File.Delete(markerPath); + else + File.WriteAllBytes(markerPath, oldMarkerBytes); + throw; + } + promoted = true; + PublisherWorkspace.RecordPublished(state, buildId); + return VerifyDestination(destination, signTool); + } + finally + { + if (!promoted) + { + foreach (var artifact in createdArtifacts.AsEnumerable().Reverse()) + File.Delete(artifact); + foreach (var directory in createdDirectories.AsEnumerable().Reverse()) + { + if (Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any()) + Directory.Delete(directory); + } + } + TryDeleteDirectory(stagingRoot); + } + } + + private static JsonObject? ExistingDestinationMarker(string destination, string markerPath) + { + var entries = Directory.EnumerateFileSystemEntries(destination) + .Where(path => !Path.GetFileName(path).Equals(MarkerName, StringComparison.Ordinal)) + .ToArray(); + if (!File.Exists(markerPath)) + { + if (entries.Length > 0) + throw new PublisherException($"Refusing to publish into a non-empty unowned destination: {destination}"); + return null; + } + var marker = ReadMarker(markerPath); + if (marker["schemaVersion"]?.GetValue() != 1) + throw new PublisherException("Unsupported publisher ownership marker version."); + if (!string.Equals(marker["owner"]?.GetValue(), "Pinget.Publisher", StringComparison.Ordinal)) + throw new PublisherException("Destination marker is not a Pinget publisher ownership record."); + return marker; + } + + private static JsonObject ReadMarker(string path) + { + try + { + PublisherWorkspace.EnsureSafeExistingPath(path); + return JsonNode.Parse(File.ReadAllText(path)) as JsonObject + ?? throw new PublisherException($"Invalid publisher ownership marker: {path}"); + } + catch (PublisherException) { throw; } + catch (Exception ex) { throw new PublisherException($"Cannot read publisher ownership marker '{path}': {ex.Message}", ex); } + } + + private static JsonObject ReadSnapshotArtifacts(string buildPath) + { + var metadata = JsonNode.Parse(File.ReadAllText(Path.Combine(buildPath, "snapshot.json"))) as JsonObject + ?? throw new PublisherException("Invalid snapshot metadata."); + return metadata["artifacts"] as JsonObject + ?? throw new PublisherException("Snapshot artifact inventory is missing."); + } + + private static string ReadSnapshotIndexHash(string buildPath) + { + var metadata = JsonNode.Parse(File.ReadAllText(Path.Combine(buildPath, "snapshot.json"))) as JsonObject + ?? throw new PublisherException("Invalid snapshot metadata."); + return metadata["indexHash"]?.GetValue() + ?? throw new PublisherException("Snapshot index hash is missing."); + } + + private static void VerifyMsixContents(string packagePath, JsonObject marker) + { + try + { + using var archive = ZipFile.OpenRead(packagePath); + var manifestEntry = archive.GetEntry("AppxManifest.xml") + ?? throw new PublisherException("MSIX does not contain AppxManifest.xml."); + var indexEntry = archive.GetEntry("Public/index.db") + ?? throw new PublisherException("MSIX does not expose Public/index.db."); + using (var indexStream = indexEntry.Open()) + { + var actualIndexHash = Convert.ToHexString(SHA256.HashData(indexStream)).ToLowerInvariant(); + if (!string.Equals(actualIndexHash, marker["indexHash"]?.GetValue(), StringComparison.Ordinal)) + throw new PublisherException("MSIX contains a v2 index different from the published build."); + } + + using var manifestStream = manifestEntry.Open(); + var xml = XDocument.Load(manifestStream); + XNamespace foundation = "http://schemas.microsoft.com/appx/manifest/foundation/windows10"; + XNamespace uap3 = "http://schemas.microsoft.com/appx/manifest/uap/windows10/3"; + var identity = xml.Root?.Element(foundation + "Identity") + ?? throw new PublisherException("MSIX AppxManifest.xml has no Identity."); + if (!string.Equals((string?)identity.Attribute("Name"), marker["packageName"]?.GetValue(), StringComparison.Ordinal) + || !string.Equals((string?)identity.Attribute("Publisher"), marker["publisher"]?.GetValue(), StringComparison.Ordinal) + || !string.Equals((string?)identity.Attribute("Version"), marker["packageVersion"]?.GetValue(), StringComparison.Ordinal)) + throw new PublisherException("MSIX identity does not match the publisher ownership marker."); + var extension = xml.Descendants(uap3 + "AppExtension").SingleOrDefault(); + if (!string.Equals((string?)extension?.Attribute("Name"), "com.microsoft.winget.source", StringComparison.Ordinal) + || !string.Equals((string?)extension?.Attribute("PublicFolder"), "Public", StringComparison.Ordinal)) + throw new PublisherException("MSIX does not expose the expected WinGet preindexed source extension."); + } + catch (PublisherException) { throw; } + catch (Exception ex) { throw new PublisherException($"Could not verify source MSIX contents: {ex.Message}", ex); } + } + + private static void ValidateIdentity(string packageName, string publisher) + { + if (!PackageNamePattern().IsMatch(packageName)) + throw new PublisherException("Package identity name must contain only letters, digits, and periods."); + if (string.IsNullOrWhiteSpace(publisher) || publisher.Any(char.IsControl) || publisher.Contains('<') || publisher.Contains('>')) + throw new PublisherException("Publisher must be a non-empty certificate subject without control or XML delimiter characters."); + } + + private static Version ParsePackageVersion(string value) + { + var components = value.Split('.'); + if (components.Length != 4 || components.Any(part => !ushort.TryParse(part, out _)) + || !Version.TryParse(value, out var version) || version.Build < 0 || version.Revision < 0) + throw new PublisherException("Source MSIX package version must have four numeric components (for example 1.0.0.0)."); + return version; + } + + private static void ValidateCertificate(string pfxPath, string password, string publisher) + { + var fullPath = Path.GetFullPath(pfxPath); + if (!File.Exists(fullPath)) + throw new PublisherException($"PFX file does not exist: {fullPath}"); + if (new FileInfo(fullPath).Length > 16 * 1024 * 1024) + throw new PublisherException("PFX file exceeds the 16 MiB limit."); + try + { +#pragma warning disable SYSLIB0057 + using var certificate = new X509Certificate2(fullPath, password, X509KeyStorageFlags.EphemeralKeySet); +#pragma warning restore SYSLIB0057 + var now = DateTime.UtcNow; + if (!certificate.HasPrivateKey) + throw new PublisherException("The PFX certificate has no private key."); + if (now < certificate.NotBefore.ToUniversalTime() || now > certificate.NotAfter.ToUniversalTime()) + throw new PublisherException("The signing certificate is not currently valid."); + if (!string.Equals(certificate.Subject, publisher, StringComparison.OrdinalIgnoreCase)) + throw new PublisherException($"Signing certificate subject '{certificate.Subject}' does not match publisher '{publisher}'."); + var eku = certificate.Extensions.OfType().FirstOrDefault(); + if (eku is null || !eku.EnhancedKeyUsages.Cast().Any(oid => oid.Value == "1.3.6.1.5.5.7.3.3")) + throw new PublisherException("The signing certificate must include the Code Signing enhanced key usage."); + } + catch (PublisherException) { throw; } + catch (Exception ex) { throw new PublisherException($"Could not load PFX certificate: {ex.Message}", ex); } + } + + private static void WriteAppxManifest(string path, PublisherPublishOptions options, Version version) + { + var name = System.Security.SecurityElement.Escape(options.PackageName); + var publisher = System.Security.SecurityElement.Escape(options.Publisher); + var versionText = version.ToString(); + var xml = $""" + + + + + Pinget preindexed source + Pinget source publisher + Assets\StoreLogo.png + + + + + + + + + + + + + + + + + + + """; + File.WriteAllText(path, xml.Replace("\r\n", "\n", StringComparison.Ordinal), new UTF8Encoding(false)); + } + + private static void CopyAtomically(string source, string destination, bool overwrite = true) + { + var temporary = destination + $".{Guid.NewGuid():N}.tmp"; + try + { + File.Copy(source, temporary); + File.Move(temporary, destination, overwrite); + } + finally + { + if (File.Exists(temporary)) + File.Delete(temporary); + } + } + + private static void WriteJsonAtomically(string path, JsonObject value) + { + var temporary = path + $".{Guid.NewGuid():N}.tmp"; + try + { + File.WriteAllText(temporary, value.ToJsonString(new JsonSerializerOptions { WriteIndented = true }), new UTF8Encoding(false)); + File.Move(temporary, path, overwrite: true); + } + finally + { + if (File.Exists(temporary)) + File.Delete(temporary); + } + } + + private static void RunTool(string executable, IReadOnlyList arguments, string operation, string? sensitiveValue = null) + { + var startInfo = new ProcessStartInfo + { + FileName = executable, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + foreach (var argument in arguments) + startInfo.ArgumentList.Add(argument); + using var process = new Process { StartInfo = startInfo }; + try + { + process.Start(); + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + process.WaitForExit(); + var stdout = stdoutTask.GetAwaiter().GetResult(); + var stderr = stderrTask.GetAwaiter().GetResult(); + if (process.ExitCode != 0) + { + var output = stderr + stdout; + if (!string.IsNullOrEmpty(sensitiveValue)) + output = output.Replace(sensitiveValue, "[REDACTED]", StringComparison.Ordinal); + throw new PublisherException($"{operation} failed with exit code {process.ExitCode}: {Limit(output)}"); + } + } + catch (PublisherException) { throw; } + catch (Exception ex) { throw new PublisherException($"Could not start {operation}: {ex.Message}", ex); } + } + + private static string? FindTool(string? explicitPath, string executable) + { + if (!string.IsNullOrWhiteSpace(explicitPath)) + { + var path = Path.GetFullPath(explicitPath); + if (!File.Exists(path)) + throw new PublisherException($"Tool does not exist: {path}"); + return path; + } + var pathValue = Environment.GetEnvironmentVariable("PATH") ?? ""; + foreach (var directory in pathValue.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)) + { + var candidate = Path.Combine(directory.Trim('"'), executable); + if (File.Exists(candidate)) + return candidate; + } + var kits = Environment.GetEnvironmentVariable("ProgramFiles(x86)"); + if (OperatingSystem.IsWindows() && kits is not null) + { + var root = Path.Combine(kits, "Windows Kits", "10", "bin"); + if (Directory.Exists(root)) + { + var architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(); + foreach (var candidate in Directory.EnumerateFiles(root, executable, SearchOption.AllDirectories) + .Where(path => string.Equals(Path.GetFileName(Path.GetDirectoryName(path)), architecture, StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(path => Version.TryParse(Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(path))), out var version) ? version : new Version()) + .ThenBy(path => path, StringComparer.Ordinal)) + return candidate; + } + } + return null; + } + + private static string HashFile(string path) => + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))).ToLowerInvariant(); + + private static string Limit(string text) => text.Length <= 4000 ? text.Trim() : text[..4000].Trim(); + + private static void TryDeleteDirectory(string path) + { + try { if (Directory.Exists(path)) Directory.Delete(path, recursive: true); } + catch { } + } + + [GeneratedRegex(@"^[A-Za-z0-9]+(?:\.[A-Za-z0-9]+)*$", RegexOptions.CultureInvariant)] + private static partial Regex PackageNamePattern(); +} diff --git a/dotnet/src/Devolutions.Pinget.Publisher/PublisherWorkspace.cs b/dotnet/src/Devolutions.Pinget.Publisher/PublisherWorkspace.cs new file mode 100644 index 0000000..5ffbabb --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/PublisherWorkspace.cs @@ -0,0 +1,586 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Encodings.Web; +using System.Text.RegularExpressions; +using Microsoft.Data.Sqlite; +using YamlDotNet.RepresentationModel; + +namespace Pinget.Publisher; + +public sealed partial class PublisherWorkspace +{ + private const int StateSchemaVersion = 1; + internal const int ArtifactFormatVersion = 2; + private readonly string _stateDirectory; + + public PublisherWorkspace(string stateDirectory) + { + _stateDirectory = Path.GetFullPath(stateDirectory); + } + + public PublisherBuildResult Build(PublisherBuildOptions options) + { + var input = Path.GetFullPath(options.InputDirectory); + var state = Path.GetFullPath(options.StateDirectory); + EnsureDisjoint(input, state, "manifest", "state"); + EnsureSafeExistingPath(input); + EnsureSafeExistingPath(state); + if (File.Exists(Path.Combine(state, "publisher.db"))) + { + using var existingState = OpenState(state, create: false); + } + var scan = ManifestScanner.Scan(input); + Directory.CreateDirectory(state); + EnsureSafeExistingPath(state); + using var writerLock = AcquireLock(state); + using var stateCheck = OpenState(state, create: true); + Directory.CreateDirectory(Path.Combine(state, "builds")); + EnsureSafeExistingPath(Path.Combine(state, "builds")); + var buildId = ComputeBuildId(scan.Inventory); + var snapshot = Path.Combine(state, "builds", buildId); + var changed = 0; + var removed = 0; + var previousInventory = TryGetLatestInventory(state); + if (previousInventory is not null) + { + changed = scan.Inventory.Count(pair => previousInventory.TryGetValue(pair.Key, out var hash) && !hash.Equals(pair.Value, StringComparison.Ordinal)); + changed += scan.Inventory.Keys.Count(path => !previousInventory.ContainsKey(path)); + removed = previousInventory.Keys.Count(path => !scan.Inventory.ContainsKey(path)); + } + + if (!Directory.Exists(snapshot)) + { + var staging = Path.Combine(state, "builds", $".staging-{Guid.NewGuid():N}"); + Directory.CreateDirectory(staging); + try + { + WriteSnapshot(staging, buildId, scan); + Directory.Move(staging, snapshot); + } + catch + { + TryDeleteDirectory(staging); + throw; + } + } + else + { + VerifySnapshotDirectory(snapshot, buildId); + } + + using var connection = OpenState(state, create: true); + using var transaction = connection.BeginTransaction(); + using (var insert = connection.CreateCommand()) + { + insert.Transaction = transaction; + insert.CommandText = "INSERT OR REPLACE INTO builds(id,inventory,summary) VALUES($id,$inventory,$summary)"; + insert.Parameters.AddWithValue("$id", buildId); + insert.Parameters.AddWithValue("$inventory", WriteInventory(scan.Inventory)); + var snapshotMetadata = JsonNode.Parse(File.ReadAllText(Path.Combine(snapshot, "snapshot.json")))!.AsObject(); + var summary = new JsonObject + { + ["manifestCount"] = scan.Manifests.Count, + ["packageCount"] = scan.Packages.Count, + ["createdUtc"] = snapshotMetadata["createdUtc"]!.GetValue(), + ["snapshot"] = Path.GetRelativePath(state, snapshot).Replace('\\', '/'), + }; + insert.Parameters.AddWithValue("$summary", summary.ToJsonString()); + insert.ExecuteNonQuery(); + } + SetMetadata(connection, transaction, "schemaVersion", StateSchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + SetMetadata(connection, transaction, "lastBuildId", buildId); + transaction.Commit(); + return new PublisherBuildResult(buildId, snapshot, scan.Manifests.Count, scan.Packages.Count, changed, removed); + } + + public PublisherDiffResult Diff(string inputDirectory) + { + EnsureSafeExistingPath(_stateDirectory); + EnsureSafeExistingPath(Path.GetFullPath(inputDirectory)); + EnsureDisjoint(Path.GetFullPath(inputDirectory), _stateDirectory, "manifest", "state"); + using var connection = OpenState(_stateDirectory, create: false); + var current = ManifestScanner.Scan(Path.GetFullPath(inputDirectory), generateArtifacts: false).Inventory; + var latestBuildId = GetMetadata(connection, "lastBuildId"); + var publishedBuildId = GetMetadata(connection, "lastPublishedBuildId"); + if (latestBuildId is null) + return new PublisherDiffResult(null, false, current.Keys.Order(StringComparer.Ordinal).ToArray(), [], []); + var previous = ReadInventory(connection, latestBuildId) + ?? throw new PublisherException($"State metadata references missing build '{latestBuildId}'."); + var added = current.Keys.Where(path => !previous.ContainsKey(path)).Order(StringComparer.Ordinal).ToArray(); + var changed = current.Keys.Where(path => previous.TryGetValue(path, out var hash) && !hash.Equals(current[path], StringComparison.Ordinal)).Order(StringComparer.Ordinal).ToArray(); + var removed = previous.Keys.Where(path => !current.ContainsKey(path)).Order(StringComparer.Ordinal).ToArray(); + return new PublisherDiffResult(latestBuildId, string.Equals(publishedBuildId, latestBuildId, StringComparison.Ordinal), added, changed, removed); + } + + public PublisherStatusResult Status() + { + EnsureSafeExistingPath(_stateDirectory); + using var connection = OpenState(_stateDirectory, create: false); + var latest = GetMetadata(connection, "lastBuildId"); + var published = GetMetadata(connection, "lastPublishedBuildId"); + using var count = connection.CreateCommand(); + count.CommandText = "SELECT COUNT(*) FROM builds"; + var buildCount = Convert.ToInt32(count.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture); + string? timestamp = null; + if (latest is not null) + { + var summary = ReadBuildSummary(connection, latest); + timestamp = summary?["createdUtc"]?.GetValue(); + } + return new PublisherStatusResult(latest, published, buildCount, timestamp); + } + + public PublisherVerifyResult Verify(string? buildId = null) + { + EnsureSafeExistingPath(_stateDirectory); + using var connection = OpenState(_stateDirectory, create: false); + buildId ??= GetMetadata(connection, "lastBuildId") + ?? throw new PublisherException("Publisher state has no successful build."); + if (!BuildIdPattern().IsMatch(buildId)) + throw new PublisherException($"Invalid build ID: '{buildId}'."); + var stateInventory = ReadInventory(connection, buildId) + ?? throw new PublisherException($"Publisher state does not contain build '{buildId}'."); + var snapshot = Path.Combine(_stateDirectory, "builds", buildId); + VerifySnapshotDirectory(snapshot, buildId); + var snapshotMetadata = JsonNode.Parse(File.ReadAllText(Path.Combine(snapshot, "snapshot.json"))) as JsonObject; + var snapshotInventory = snapshotMetadata?["inventory"] as JsonObject + ?? throw new PublisherException("Snapshot input inventory is missing."); + if (stateInventory.Count != snapshotInventory.Count + || stateInventory.Any(item => !string.Equals(snapshotInventory[item.Key]?.GetValue(), item.Value, StringComparison.Ordinal))) + throw new PublisherException("Snapshot input inventory does not match publisher state."); + var summary = ReadBuildSummary(connection, buildId) + ?? throw new PublisherException("Publisher build summary is missing."); + foreach (var key in new[] { "manifestCount", "packageCount", "createdUtc" }) + if (!JsonNode.DeepEquals(summary[key], snapshotMetadata![key])) + throw new PublisherException($"Snapshot '{key}' does not match publisher build summary."); + return new PublisherVerifyResult(true, "unsigned build snapshot", CountSnapshotArtifacts(snapshot), buildId); + } + + internal static void RecordPublished(string stateDirectory, string buildId) + { + var state = Path.GetFullPath(stateDirectory); + using var connection = OpenState(state, create: true); + using var transaction = connection.BeginTransaction(); + if (ReadInventory(connection, buildId) is null) + throw new PublisherException($"Cannot record publication; build '{buildId}' is not in this state."); + SetMetadata(connection, transaction, "lastPublishedBuildId", buildId); + transaction.Commit(); + } + + internal static FileStream AcquireLock(string stateDirectory) + { + Directory.CreateDirectory(stateDirectory); + var path = Path.Combine(stateDirectory, ".publisher.lock"); + EnsureSafeExistingPath(path); + try + { + return new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + } + catch (IOException ex) + { + throw new PublisherException($"Another publisher writer holds the state lock at '{path}'.", ex); + } + } + + internal static SqliteConnection OpenState(string stateDirectory, bool create) + { + var path = Path.Combine(stateDirectory, "publisher.db"); + EnsureSafeExistingPath(stateDirectory); + var exists = File.Exists(path); + if (exists && (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException("Publisher state database cannot be a reparse point."); + if (!exists && !create) + throw new PublisherException($"Publisher state does not exist: {path}"); + Directory.CreateDirectory(stateDirectory); + if (!exists) + { + try + { + using var reserved = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None); + } + catch (IOException ex) + { + throw new PublisherException($"Cannot exclusively create publisher state '{path}'; retry without replacing the existing database.", ex); + } + } + var connection = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = create ? SqliteOpenMode.ReadWrite : SqliteOpenMode.ReadOnly, + Cache = SqliteCacheMode.Private, + Pooling = false, + }.ToString()); + try + { + connection.Open(); + if (create && !exists) + { + using var command = connection.CreateCommand(); + command.CommandText = WinGetV2IndexWriter.EmbeddedSql("state-schema.sql"); + command.ExecuteNonQuery(); + using var transaction = connection.BeginTransaction(); + SetMetadata(connection, transaction, "schemaVersion", StateSchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + SetMetadata(connection, transaction, "artifactFormatVersion", ArtifactFormatVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + transaction.Commit(); + } + if (!HasTable(connection, "metadata") || !HasTable(connection, "builds")) + throw new PublisherException($"Existing database is not a publisher state database: '{path}'."); + using (var tables = connection.CreateCommand()) + { + tables.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"; + using var reader = tables.ExecuteReader(); + var names = new HashSet(StringComparer.Ordinal); + while (reader.Read()) + names.Add(reader.GetString(0)); + if (!names.SetEquals(new[] { "metadata", "builds" })) + throw new PublisherException("Existing publisher state has unexpected tables."); + } + using (var columns = connection.CreateCommand()) + { + columns.CommandText = "SELECT id,inventory,summary FROM builds LIMIT 0"; + using var reader = columns.ExecuteReader(); + } + using (var version = connection.CreateCommand()) + { + version.CommandText = "PRAGMA user_version"; + if (Convert.ToInt32(version.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture) != StateSchemaVersion) + throw new PublisherException($"Unsupported publisher state schema in '{path}'."); + } + if (!string.Equals(GetMetadata(connection, "schemaVersion"), StateSchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture), StringComparison.Ordinal)) + throw new PublisherException($"Unsupported publisher state metadata version in '{path}'."); + ValidateArtifactFormatVersion(GetMetadata(connection, "artifactFormatVersion"), "publisher state"); + return connection; + } + catch + { + connection.Dispose(); + throw; + } + } + + internal static void ValidateArtifactFormatVersion(string? value, string artifact) + { + if (!string.Equals(value, ArtifactFormatVersion.ToString(System.Globalization.CultureInfo.InvariantCulture), StringComparison.Ordinal)) + throw new PublisherException($"Unsupported {artifact} artifactFormatVersion '{value ?? "missing (legacy format 1)"}'; format {ArtifactFormatVersion} is required. Use a fresh --state directory. Existing files and snapshots are preserved; automatic state migration is not supported. Existing publication destinations can receive a new, explicitly higher package version."); + } + + internal static string? GetMetadata(SqliteConnection connection, string key) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT value FROM metadata WHERE name=$name"; + command.Parameters.AddWithValue("$name", key); + return command.ExecuteScalar() as string; + } + + internal static void SetMetadata(SqliteConnection connection, SqliteTransaction transaction, string key, string value) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = "INSERT INTO metadata(name,value) VALUES($name,$value) ON CONFLICT(name) DO UPDATE SET value=excluded.value"; + command.Parameters.AddWithValue("$name", key); + command.Parameters.AddWithValue("$value", value); + command.ExecuteNonQuery(); + } + + internal static Dictionary? ReadInventory(SqliteConnection connection, string buildId) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT inventory FROM builds WHERE id=$id"; + command.Parameters.AddWithValue("$id", buildId); + if (command.ExecuteScalar() is not string json) + return null; + var result = new SortedDictionary(StringComparer.Ordinal); + using var document = JsonDocument.Parse(json); + foreach (var item in document.RootElement.EnumerateObject()) + result.Add(item.Name, item.Value.GetString()!); + return result.ToDictionary(StringComparer.Ordinal); + } + + internal static JsonObject? ReadBuildSummary(SqliteConnection connection, string buildId) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT summary FROM builds WHERE id=$id"; + command.Parameters.AddWithValue("$id", buildId); + return command.ExecuteScalar() is string json ? JsonNode.Parse(json) as JsonObject : null; + } + + private static Dictionary? TryGetLatestInventory(string stateDirectory) + { + var path = Path.Combine(stateDirectory, "publisher.db"); + if (!File.Exists(path)) + return null; + using var connection = OpenState(stateDirectory, create: false); + var buildId = GetMetadata(connection, "lastBuildId"); + return buildId is null ? null : ReadInventory(connection, buildId); + } + + private static void WriteSnapshot(string directory, string buildId, ScanResult scan) + { + var feed = Path.Combine(directory, "feed"); + Directory.CreateDirectory(feed); + var artifactHashes = new SortedDictionary(StringComparer.Ordinal); + foreach (var manifest in scan.Manifests) + { + var path = ResolveContained(feed, manifest.RelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, manifest.Content); + artifactHashes.Add(manifest.RelativePath, manifest.Hash); + } + foreach (var package in scan.Packages) + { + var sidecarHash = Convert.ToHexString(package.PackageHash).ToLowerInvariant(); + var sidecarRelative = $"packages/{package.Id}/{sidecarHash[..8]}/versionData.mszyml"; + var sidecarPath = ResolveContained(feed, sidecarRelative); + Directory.CreateDirectory(Path.GetDirectoryName(sidecarPath)!); + File.WriteAllBytes(sidecarPath, package.CompressedVersionData); + artifactHashes.Add(sidecarRelative, sidecarHash); + } + var indexPath = Path.Combine(directory, "Public", "index.db"); + WinGetV2IndexWriter.Write(indexPath, scan.Packages, Guid.NewGuid().ToString("D")); + var snapshot = new JsonObject + { + ["schemaVersion"] = StateSchemaVersion, + ["artifactFormatVersion"] = ArtifactFormatVersion, + ["buildId"] = buildId, + ["createdUtc"] = DateTimeOffset.UtcNow.ToString("O"), + ["manifestCount"] = scan.Manifests.Count, + ["packageCount"] = scan.Packages.Count, + ["indexHash"] = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(indexPath))).ToLowerInvariant(), + ["inventory"] = JsonNode.Parse(WriteInventory(scan.Inventory)), + ["artifacts"] = JsonNode.Parse(WriteInventory(artifactHashes)), + }; + File.WriteAllText(Path.Combine(directory, "snapshot.json"), snapshot.ToJsonString(new JsonSerializerOptions { WriteIndented = true }), new UTF8Encoding(false)); + } + + private static void VerifySnapshotDirectory(string snapshot, string expectedBuildId) + { + if (!Directory.Exists(snapshot)) + throw new PublisherException($"Build snapshot does not exist: {snapshot}"); + EnsureSafeExistingPath(snapshot); + var metadataPath = Path.Combine(snapshot, "snapshot.json"); + EnsureSafeExistingPath(metadataPath); + if (!File.Exists(metadataPath)) + throw new PublisherException($"Build snapshot metadata is missing: {metadataPath}"); + var metadata = JsonNode.Parse(File.ReadAllText(metadataPath)) as JsonObject + ?? throw new PublisherException($"Invalid build snapshot metadata: {metadataPath}"); + if (!string.Equals(metadata["buildId"]?.GetValue(), expectedBuildId, StringComparison.Ordinal)) + throw new PublisherException($"Snapshot identity does not match expected build '{expectedBuildId}'."); + if (metadata["schemaVersion"]?.GetValue() != StateSchemaVersion) + throw new PublisherException("Unsupported snapshot schema version."); + ValidateArtifactFormatVersion(metadata["artifactFormatVersion"]?.ToJsonString(), "snapshot"); + if (metadata["inventory"] is not JsonObject inputInventory) + throw new PublisherException("Snapshot input inventory is missing."); + var inventory = inputInventory.ToDictionary(p => p.Key, p => p.Value!.GetValue(), StringComparer.Ordinal); + if (!string.Equals(ComputeBuildId(inventory), expectedBuildId, StringComparison.Ordinal)) + throw new PublisherException("Snapshot build ID does not match its input inventory."); + var feed = Path.Combine(snapshot, "feed"); + if (metadata["artifacts"] is not JsonObject artifacts) + throw new PublisherException("Snapshot artifact inventory is missing."); + var actualPaths = EnumerateSafeFiles(feed).Select(p => Path.GetRelativePath(feed, p).Replace('\\', '/')).ToHashSet(StringComparer.Ordinal); + if (!actualPaths.SetEquals(artifacts.Select(p => p.Key))) + throw new PublisherException("Snapshot feed does not match its exact artifact inventory."); + foreach (var entry in artifacts) + { + var path = ResolveContained(feed, entry.Key); + if (!File.Exists(path)) + throw new PublisherException($"Snapshot artifact is missing: {entry.Key}"); + var expected = entry.Value?.GetValue(); + var actual = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))).ToLowerInvariant(); + if (!string.Equals(expected, actual, StringComparison.Ordinal)) + throw new PublisherException($"Snapshot artifact hash mismatch: {entry.Key}"); + } + var indexPath = Path.Combine(snapshot, "Public", "index.db"); + EnsureSafeExistingPath(indexPath); + if (!File.Exists(indexPath)) + throw new PublisherException("Snapshot v2 index is missing."); + var indexHash = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(indexPath))).ToLowerInvariant(); + if (!string.Equals(indexHash, metadata["indexHash"]?.GetValue(), StringComparison.Ordinal)) + throw new PublisherException("Snapshot v2 index hash does not match snapshot metadata."); + using var database = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = indexPath, Mode = SqliteOpenMode.ReadOnly, Pooling = false }.ToString()); + database.Open(); + using (var schema = database.CreateCommand()) + { + schema.CommandText = "SELECT name,value FROM metadata WHERE name IN ('majorVersion','minorVersion')"; + using var schemaReader = schema.ExecuteReader(); + var versions = new Dictionary(StringComparer.Ordinal); + while (schemaReader.Read()) + versions[schemaReader.GetString(0)] = schemaReader.GetString(1); + if (versions.GetValueOrDefault("majorVersion") != "2" || versions.GetValueOrDefault("minorVersion") != "0") + throw new PublisherException("Unsupported snapshot index schema; expected 2.0."); + } + using var integrity = database.CreateCommand(); + integrity.CommandText = "PRAGMA integrity_check"; + if (!string.Equals(integrity.ExecuteScalar()?.ToString(), "ok", StringComparison.Ordinal)) + throw new PublisherException("Snapshot v2 index failed SQLite integrity_check."); + + using var packages = database.CreateCommand(); + packages.CommandText = "SELECT id,latest_version,hash FROM packages"; + using var reader = packages.ExecuteReader(); + while (reader.Read()) + { + var id = reader.GetString(0); + var latestVersion = reader.GetString(1); + var hash = (byte[])reader[2]; + if (hash.Length != 32) + throw new PublisherException($"Package '{id}' has an invalid sidecar hash length."); + var hashHex = Convert.ToHexString(hash).ToLowerInvariant(); + var relative = $"packages/{id}/{hashHex[..8]}/versionData.mszyml"; + var sidecar = ResolveContained(feed, relative); + if (!File.Exists(sidecar)) + throw new PublisherException($"Package '{id}' has a missing version-data sidecar."); + var sidecarBytes = File.ReadAllBytes(sidecar); + if (!CryptographicOperations.FixedTimeEquals(SHA256.HashData(sidecarBytes), hash)) + throw new PublisherException($"Package '{id}' has a missing or invalid version-data sidecar."); + VerifyVersionData(feed, id, latestVersion, MsZipCompressor.Decompress(sidecarBytes)); + } + + } + + internal static IEnumerable EnumerateSafeFiles(string root) + { + EnsureSafeExistingPath(root); + var pending = new Stack(); + pending.Push(root); + while (pending.TryPop(out var directory)) + foreach (var entry in Directory.EnumerateFileSystemEntries(directory)) + { + if ((File.GetAttributes(entry) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Reparse point is not allowed: {entry}"); + if (Directory.Exists(entry)) + pending.Push(entry); + else + yield return entry; + } + } + private static void VerifyVersionData(string feed, string packageId, string latestVersion, byte[] bytes) + { + try + { + using var reader = new StringReader(new UTF8Encoding(false, true).GetString(bytes)); + var stream = new YamlStream(); + stream.Load(reader); + if (stream.Documents.Count != 1 || stream.Documents[0].RootNode is not YamlMappingNode root) + throw new PublisherException($"Package '{packageId}' has malformed version data."); + var schema = root.Children.FirstOrDefault(pair => pair.Key is YamlScalarNode scalar && scalar.Value == "sV").Value as YamlScalarNode; + if (schema?.Value != "1.0") + throw new PublisherException($"Package '{packageId}' uses an unsupported version-data schema."); + var versionsNode = root.Children.FirstOrDefault(pair => pair.Key is YamlScalarNode scalar && scalar.Value == "vD").Value; + if (versionsNode is not YamlSequenceNode versions) + throw new PublisherException($"Package '{packageId}' version data is missing vD."); + var latestFound = false; + var seenVersions = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var node in versions.Children.OfType()) + { + var version = Scalar(node, "v"); + var relative = Scalar(node, "rP"); + var expectedHash = Scalar(node, "s256H"); + if (version is null || relative is null || expectedHash is null) + throw new PublisherException($"Package '{packageId}' has an incomplete version-data entry."); + if (!seenVersions.Add(version)) + throw new PublisherException($"Package '{packageId}' contains duplicate version-data records."); + latestFound |= string.Equals(version, latestVersion, StringComparison.OrdinalIgnoreCase); + var manifest = ResolveContained(feed, relative); + if (!File.Exists(manifest)) + throw new PublisherException($"Version data references missing manifest: {relative}"); + var actualHash = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(manifest))).ToLowerInvariant(); + if (!string.Equals(actualHash, expectedHash, StringComparison.Ordinal)) + throw new PublisherException($"Version data manifest hash mismatch: {relative}"); + var model = JsonNode.Parse(File.ReadAllBytes(manifest)) as JsonObject; + if (model?["PackageIdentifier"]?.GetValue() != packageId || model["PackageVersion"]?.GetValue() != version) + throw new PublisherException($"Version data manifest identity mismatch: {relative}"); + } + if (!latestFound) + throw new PublisherException($"Package '{packageId}' version data does not include latest version '{latestVersion}'."); + } + catch (PublisherException) { throw; } + catch (Exception ex) { throw new PublisherException($"Could not verify version data for package '{packageId}': {ex.Message}", ex); } + } + + private static string? Scalar(YamlMappingNode mapping, string name) => + mapping.Children.FirstOrDefault(pair => pair.Key is YamlScalarNode key && key.Value == name).Value is YamlScalarNode value + ? value.Value + : null; + + private static int CountSnapshotArtifacts(string snapshot) => + Directory.EnumerateFiles(Path.Combine(snapshot, "feed"), "*", SearchOption.AllDirectories).Count() + 1; + + private static string ComputeBuildId(IReadOnlyDictionary inventory) + { + var canonical = "pinget-publisher-state-v2\n" + WriteInventory(inventory); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant(); + } + + private static string WriteInventory(IReadOnlyDictionary inventory) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping })) + { + writer.WriteStartObject(); + foreach (var entry in inventory.OrderBy(x => x.Key, StringComparer.Ordinal)) + writer.WriteString(entry.Key, entry.Value); + writer.WriteEndObject(); + } + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private static bool HasTable(SqliteConnection connection, string name) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=$name"; + command.Parameters.AddWithValue("$name", name); + return command.ExecuteScalar() is not null; + } + + internal static string ResolveContained(string root, string relative) + { + if (Path.IsPathRooted(relative) || relative.Split('/').Any(part => part is "" or "." or ".." + || part.Any(c => c < ' ' || "\\:*?\"<>|".Contains(c)))) + throw new PublisherException($"Unsafe publisher relative path: '{relative}'."); + var fullRoot = Path.GetFullPath(root).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; + var fullPath = Path.GetFullPath(Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar))); + if (!fullPath.StartsWith(fullRoot, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal)) + throw new PublisherException($"Path escapes publisher root: '{relative}'."); + return fullPath; + } + + internal static void EnsureDisjoint(string first, string second, string firstLabel, string secondLabel) + { + var a = Path.GetFullPath(first).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var b = Path.GetFullPath(second).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + if (IsWithin(a, b) || IsWithin(b, a)) + throw new PublisherException($"The {firstLabel} and {secondLabel} directories must not overlap."); + } + + internal static bool IsWithin(string root, string path) + { + var prefix = Path.GetFullPath(root).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + var fullPath = Path.GetFullPath(path); + return fullPath.StartsWith(prefix, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) + || string.Equals(Path.GetFullPath(root), fullPath, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); + } + + internal static void EnsureSafeExistingPath(string path) + { + if (File.Exists(path) && (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Reparse points are not allowed in publisher paths: {path}"); + var full = Path.GetFullPath(path); + var current = new DirectoryInfo(full); + while (current is not null) + { + if (current.Exists && (current.Attributes & FileAttributes.ReparsePoint) != 0) + throw new PublisherException($"Reparse points are not allowed in publisher paths: {current.FullName}"); + current = current.Parent; + } + } + + private static void TryDeleteDirectory(string path) + { + try { if (Directory.Exists(path)) Directory.Delete(path, recursive: true); } + catch { } + } + + [GeneratedRegex(@"^[a-f0-9]{64}$", RegexOptions.CultureInvariant)] + private static partial Regex BuildIdPattern(); +} diff --git a/dotnet/src/Devolutions.Pinget.Publisher/Sql/index-schema.sql b/dotnet/src/Devolutions.Pinget.Publisher/Sql/index-schema.sql new file mode 100644 index 0000000..d0ea538 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/Sql/index-schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE metadata(name TEXT PRIMARY KEY NOT NULL,value TEXT NOT NULL) WITHOUT ROWID; +CREATE TABLE packages( + rowid INTEGER PRIMARY KEY, + id TEXT NOT NULL, + name TEXT NOT NULL, + moniker TEXT, + latest_version TEXT NOT NULL, + arp_min_version TEXT, + arp_max_version TEXT, + hash BLOB +); +CREATE TABLE tags2(rowid INTEGER PRIMARY KEY,tag TEXT NOT NULL); +CREATE TABLE tags2_map(tag INTEGER NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(tag,package)) WITHOUT ROWID; +CREATE TABLE commands2(rowid INTEGER PRIMARY KEY,command TEXT NOT NULL); +CREATE TABLE commands2_map(command INTEGER NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(command,package)) WITHOUT ROWID; +CREATE TABLE pfns2(pfn TEXT NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(pfn,package)) WITHOUT ROWID; +CREATE TABLE productcodes2(productcode TEXT NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(productcode,package)) WITHOUT ROWID; +CREATE TABLE norm_names2(norm_name TEXT NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(norm_name,package)) WITHOUT ROWID; +CREATE TABLE norm_publishers2(norm_publisher TEXT NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(norm_publisher,package)) WITHOUT ROWID; +CREATE TABLE upgradecodes2(upgradecode TEXT NOT NULL,package INTEGER NOT NULL,PRIMARY KEY(upgradecode,package)) WITHOUT ROWID; diff --git a/dotnet/src/Devolutions.Pinget.Publisher/Sql/state-schema.sql b/dotnet/src/Devolutions.Pinget.Publisher/Sql/state-schema.sql new file mode 100644 index 0000000..3ae6211 --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/Sql/state-schema.sql @@ -0,0 +1,3 @@ +PRAGMA user_version=1; +CREATE TABLE metadata(name TEXT PRIMARY KEY NOT NULL,value TEXT NOT NULL) WITHOUT ROWID; +CREATE TABLE builds(id TEXT PRIMARY KEY NOT NULL,inventory TEXT NOT NULL,summary TEXT NOT NULL) WITHOUT ROWID; diff --git a/dotnet/src/Devolutions.Pinget.Publisher/WinGetV2IndexWriter.cs b/dotnet/src/Devolutions.Pinget.Publisher/WinGetV2IndexWriter.cs new file mode 100644 index 0000000..61a035f --- /dev/null +++ b/dotnet/src/Devolutions.Pinget.Publisher/WinGetV2IndexWriter.cs @@ -0,0 +1,166 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Data.Sqlite; + +namespace Pinget.Publisher; + +internal static class WinGetV2IndexWriter +{ + public static void Write(string path, IReadOnlyList packages, string databaseIdentifier) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + using var connection = new SqliteConnection(new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = SqliteOpenMode.ReadWriteCreate, + Cache = SqliteCacheMode.Private, + Pooling = false, + }.ToString()); + connection.Open(); + using var transaction = connection.BeginTransaction(); + using (var schema = connection.CreateCommand()) + { + schema.Transaction = transaction; + schema.CommandText = EmbeddedSql("index-schema.sql"); + schema.ExecuteNonQuery(); + } + + using (var metadata = connection.CreateCommand()) + { + metadata.Transaction = transaction; + metadata.CommandText = "INSERT INTO metadata(name,value) VALUES($name,$value)"; + var name = metadata.Parameters.Add("$name", SqliteType.Text); + var value = metadata.Parameters.Add("$value", SqliteType.Text); + void Set(string key, string data) + { + name.Value = key; + value.Value = data; + metadata.ExecuteNonQuery(); + } + Set("majorVersion", "2"); + Set("minorVersion", "0"); + Set("databaseIdentifier", $"{{{databaseIdentifier.ToUpperInvariant()}}}"); + Set("lastwritetime", DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)); + } + + foreach (var package in packages.OrderBy(p => p.Id, StringComparer.OrdinalIgnoreCase)) + { + var packageId = InsertPackage(connection, transaction, package); + InsertMap(connection, transaction, "tags2", "tags2_map", "tag", package.Tags, packageId); + InsertMap(connection, transaction, "commands2", "commands2_map", "command", package.Commands, packageId); + InsertValues(connection, transaction, "pfns2", "pfn", package.Pfns, packageId); + InsertValues(connection, transaction, "productcodes2", "productcode", package.ProductCodes, packageId); + InsertValues(connection, transaction, "upgradecodes2", "upgradecode", package.UpgradeCodes, packageId); + InsertValues(connection, transaction, "norm_names2", "norm_name", package.NormalizedNames, packageId); + InsertValues(connection, transaction, "norm_publishers2", "norm_publisher", package.NormalizedPublishers, packageId); + } + transaction.Commit(); + connection.Close(); + using var check = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = path, Mode = SqliteOpenMode.ReadOnly, Pooling = false }.ToString()); + check.Open(); + using var integrity = check.CreateCommand(); + integrity.CommandText = "PRAGMA integrity_check"; + if (!string.Equals(integrity.ExecuteScalar()?.ToString(), "ok", StringComparison.Ordinal)) + throw new PublisherException("Generated source index failed SQLite integrity_check."); + } + + private static long InsertPackage(SqliteConnection connection, SqliteTransaction transaction, IndexedPackage package) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + INSERT INTO packages(id,name,moniker,latest_version,arp_min_version,arp_max_version,hash) + VALUES($id,$name,$moniker,$version,$arpMin,$arpMax,$hash); + SELECT last_insert_rowid(); + """; + command.Parameters.AddWithValue("$id", package.Id); + command.Parameters.AddWithValue("$name", package.Name); + command.Parameters.AddWithValue("$moniker", (object?)package.Moniker ?? DBNull.Value); + command.Parameters.AddWithValue("$version", package.LatestVersion); + command.Parameters.AddWithValue("$arpMin", (object?)package.ArpMinimumVersion ?? DBNull.Value); + command.Parameters.AddWithValue("$arpMax", (object?)package.ArpMaximumVersion ?? DBNull.Value); + command.Parameters.Add("$hash", SqliteType.Blob).Value = package.PackageHash; + return (long)command.ExecuteScalar()!; + } + + private static void InsertMap( + SqliteConnection connection, + SqliteTransaction transaction, + string valueTable, + string mapTable, + string column, + IReadOnlyList values, + long packageId) + { + foreach (var value in values) + { + long valueId; + using (var find = connection.CreateCommand()) + { + find.Transaction = transaction; + find.CommandText = $"SELECT rowid FROM {valueTable} WHERE {column}=$value COLLATE NOCASE LIMIT 1"; + find.Parameters.AddWithValue("$value", value); + var existing = find.ExecuteScalar(); + if (existing is not null) + { + valueId = (long)existing; + using var canonical = connection.CreateCommand(); + canonical.Transaction = transaction; + canonical.CommandText = $"SELECT {column} FROM {valueTable} WHERE rowid=$id"; + canonical.Parameters.AddWithValue("$id", valueId); + if (StringComparer.Ordinal.Compare(value, canonical.ExecuteScalar() as string) < 0) + { + canonical.CommandText = $"UPDATE {valueTable} SET {column}=$value WHERE rowid=$id"; + canonical.Parameters.AddWithValue("$value", value); + canonical.ExecuteNonQuery(); + } + } + else + { + using var insert = connection.CreateCommand(); + insert.Transaction = transaction; + insert.CommandText = $"INSERT INTO {valueTable}({column}) VALUES($value); SELECT last_insert_rowid();"; + insert.Parameters.AddWithValue("$value", value); + valueId = (long)insert.ExecuteScalar()!; + } + } + using var map = connection.CreateCommand(); + map.Transaction = transaction; + map.CommandText = $"INSERT INTO {mapTable}({column},package) VALUES($value,$package)"; + map.Parameters.AddWithValue("$value", valueId); + map.Parameters.AddWithValue("$package", packageId); + map.ExecuteNonQuery(); + } + } + + private static void InsertValues( + SqliteConnection connection, + SqliteTransaction transaction, + string table, + string column, + IReadOnlyList values, + long packageId) + { + foreach (var value in values) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = $"INSERT OR IGNORE INTO {table}({column},package) VALUES($value,$package)"; + command.Parameters.AddWithValue("$value", value); + command.Parameters.AddWithValue("$package", packageId); + command.ExecuteNonQuery(); + } + } + + internal static string EmbeddedSql(string name) + { + var assembly = typeof(WinGetV2IndexWriter).Assembly; + var resource = assembly.GetManifestResourceNames() + .SingleOrDefault(n => n.EndsWith($".{name}", StringComparison.Ordinal)) + ?? throw new PublisherException($"Embedded publisher SQL resource is missing: {name}"); + using var stream = assembly.GetManifestResourceStream(resource)!; + using var reader = new StreamReader(stream, Encoding.UTF8); + return reader.ReadToEnd(); + } +} diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 65538be..8648842 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/pinget-core", + "crates/pinget-publisher", "crates/pinget-cli", "crates/pinget-com", ] diff --git a/rust/crates/pinget-cli/Cargo.toml b/rust/crates/pinget-cli/Cargo.toml index da151d3..888430f 100644 --- a/rust/crates/pinget-cli/Cargo.toml +++ b/rust/crates/pinget-cli/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" anyhow = "1.0.102" clap = { version = "4.6.1", features = ["derive"] } pinget-core = { version = "0.14.0", path = "../pinget-core" } +pinget-publisher = { version = "0.14.0", path = "../pinget-publisher" } chrono = "0.4.44" dirs = "6.0" jsonschema = { version = "0.30", default-features = false, features = ["resolve-file"] } diff --git a/rust/crates/pinget-cli/src/main.rs b/rust/crates/pinget-cli/src/main.rs index f126a93..4b5dff6 100644 --- a/rust/crates/pinget-cli/src/main.rs +++ b/rust/crates/pinget-cli/src/main.rs @@ -2,7 +2,7 @@ use std::fmt; use std::io::{self, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Result, anyhow, bail}; use clap::{Args, Parser, Subcommand}; @@ -12,6 +12,7 @@ use pinget_core::{ SearchMatch, SearchResponse, ShowResult, SourceKind, SourceRecord, SourceUpdateResult, UninstallRequest, VersionsResult, }; +use pinget_publisher::{BuildOptions as PublisherBuildOptions, PublishOptions as PublisherPublishOptions, Signing}; const VERSION: &str = env!("CARGO_PKG_VERSION"); const UPGRADE_UNSUPPORTED_WARNING: &str = "Upgrading packages is not supported on this platform; no changes were made."; @@ -91,6 +92,86 @@ enum Commands { #[command(alias = "fix")] Repair(RepairArgs), Import(ImportArgs), + Repository { + #[command(subcommand)] + command: RepositoryCommands, + }, +} + +#[derive(Subcommand)] +enum RepositoryCommands { + Build(RepositoryBuildArgs), + Diff(RepositoryBuildArgs), + Status(RepositoryStateArgs), + Verify(RepositoryVerifyArgs), + Publish(RepositoryPublishArgs), + Doctor(RepositoryDoctorArgs), +} + +#[derive(Args, Clone)] +struct RepositoryBuildArgs { + #[arg(long = "input", visible_alias = "manifests")] + input: PathBuf, + #[arg(long)] + state: PathBuf, +} + +#[derive(Args)] +struct RepositoryStateArgs { + #[arg(long)] + state: PathBuf, +} + +#[derive(Args)] +struct RepositoryVerifyArgs { + #[arg(long, required_unless_present = "destination", conflicts_with = "destination")] + state: Option, + #[arg(long)] + build: Option, + #[arg(long, conflicts_with = "build")] + destination: Option, + #[arg(long)] + signtool: Option, +} + +#[derive(Args)] +struct RepositoryPublishArgs { + #[arg(long)] + state: PathBuf, + #[arg(long)] + destination: PathBuf, + #[arg(long)] + name: String, + #[arg(long)] + publisher: String, + #[arg(long = "version")] + package_version: String, + #[arg(long)] + build: Option, + #[arg(long = "pfx", requires = "password_env", conflicts_with = "unsigned_development")] + pfx: Option, + #[arg(long = "password-env", requires = "pfx", value_name = "ENVIRONMENT_VARIABLE")] + password_env: Option, + #[arg( + long = "unsigned-development", + required_unless_present = "pfx", + conflicts_with_all = ["pfx", "password_env"] + )] + unsigned_development: bool, + #[arg(long)] + makeappx: Option, + #[arg(long)] + signtool: Option, + #[arg(long, requires = "pfx", conflicts_with = "unsigned_development")] + timestamp_url: Option, +} + +#[derive(Args)] +struct RepositoryDoctorArgs { + #[arg(long)] + makeappx: Option, + #[arg(long)] + signtool: Option, } #[derive(Subcommand)] @@ -1041,11 +1122,158 @@ fn run() -> Result<()> { args.accept_package_agreements, )?; } + Commands::Repository { command } => run_repository_command(command, output)?, } Ok(()) } +fn run_repository_command(command: RepositoryCommands, output: OutputFormat) -> Result<()> { + match command { + RepositoryCommands::Build(args) => { + let result = pinget_publisher::build(&PublisherBuildOptions { + input: args.input, + state: args.state, + })?; + if output.is_text() { + println!( + "Built {} package(s) into snapshot {} ({} manifest(s), {} file(s)).", + result.packages, result.build_id, result.manifests, result.files + ); + } else { + print_serialized(&result, output)?; + } + } + RepositoryCommands::Diff(args) => { + let result = pinget_publisher::diff(&PublisherBuildOptions { + input: args.input, + state: args.state, + })?; + if output.is_text() { + println!( + "Added: {}; changed: {}; removed: {}; last build published: {}.", + result.added.len(), + result.changed.len(), + result.removed.len(), + result.last_build_published + ); + for path in &result.added { + println!(" added {path}"); + } + for path in &result.changed { + println!(" changed {path}"); + } + for path in &result.removed { + println!(" removed {path}"); + } + } else { + print_serialized(&result, output)?; + } + } + RepositoryCommands::Status(args) => { + let result = pinget_publisher::status(args.state)?; + if output.is_text() { + match &result.last_build { + Some(build) => println!( + "Latest build {}: {} package(s), {} manifest(s); published: {}.", + build.build_id, build.packages, build.manifests, result.last_build_published + ), + None => println!("No successful repository builds are recorded."), + } + } else { + print_serialized(&result, output)?; + } + } + RepositoryCommands::Verify(args) => { + let result = if let Some(destination) = args.destination.as_deref() { + pinget_publisher::verify_published_destination(destination, args.signtool.as_deref())? + } else { + pinget_publisher::verify( + args.state.as_deref().unwrap_or_else(|| Path::new(".")), + args.build.as_deref(), + None, + )? + }; + if output.is_text() { + println!( + "Verified {} ({} artifact(s)){}.", + result.subject, + result.artifact_count, + result + .build_id + .as_deref() + .map(|id| format!(" for build {id}")) + .unwrap_or_default() + ); + } else { + print_serialized(&result, output)?; + } + } + RepositoryCommands::Publish(args) => { + let signing = if args.unsigned_development { + Signing::UnsignedDevelopment + } else { + Signing::Pfx { + path: args.pfx.ok_or_else(|| anyhow!("normal publication requires --pfx"))?, + password_env: args + .password_env + .ok_or_else(|| anyhow!("normal publication requires --password-env"))?, + } + }; + let result = pinget_publisher::publish(&PublisherPublishOptions { + state: args.state, + build_id: args.build, + destination: args.destination, + name: args.name, + publisher: args.publisher, + version: args.package_version, + signing, + makeappx: args.makeappx, + signtool: args.signtool, + timestamp_url: args.timestamp_url, + })?; + if output.is_text() { + println!( + "Published snapshot {} to {}{}.", + result.build_id, + result.destination.display(), + if result.unsigned_development { + " (unsigned development package)" + } else { + "" + } + ); + } else { + print_serialized(&result, output)?; + } + } + RepositoryCommands::Doctor(args) => { + let result = pinget_publisher::doctor_with_tools(args.makeappx.as_deref(), args.signtool.as_deref()); + if output.is_text() { + println!( + "Native MSZIP: {}; makeappx: {}; signtool: {}; signed publishing available: {}.", + result.native_mszip_available, + result + .makeappx_path + .as_deref() + .map_or_else(|| "not found".to_owned(), |path| path.display().to_string()), + result + .signtool_path + .as_deref() + .map_or_else(|| "not found".to_owned(), |path| path.display().to_string()), + result.signed_publish + ); + if !result.native_mszip_available { + println!(" Note: MSZIP build output is currently supported only on Windows."); + } + } else { + print_serialized(&result, output)?; + } + } + } + Ok(()) +} + fn print_serialized(value: &T, output: OutputFormat) -> Result<()> { match output { OutputFormat::Text => bail!("structured output requested without a serializer"), diff --git a/rust/crates/pinget-core/src/lib.rs b/rust/crates/pinget-core/src/lib.rs index bd6ba2c..ba7de0c 100644 --- a/rust/crates/pinget-core/src/lib.rs +++ b/rust/crates/pinget-core/src/lib.rs @@ -43,6 +43,34 @@ use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken} use winreg::{RegKey, enums::*}; use zip::ZipArchive; +/// Normalized fields used by WinGet's installed-package identity correlation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogNameNormalization { + /// Name value used by the v2 `norm_names2` correlation table. + pub name: String, + /// Architecture suffix recognized by the normalizer, when present. + pub architecture: String, + /// Locale suffix recognized by the normalizer, when present. + pub locale: String, +} + +/// Normalizes a package display name with Pinget's WinGet-compatible algorithm. +#[must_use] +pub fn normalize_catalog_name(value: &str) -> CatalogNameNormalization { + let normalized = name_normalization::normalize_name(value); + CatalogNameNormalization { + name: normalized.name, + architecture: format!("{:?}", normalized.architecture).to_ascii_lowercase(), + locale: normalized.locale, + } +} + +/// Normalizes a publisher with Pinget's WinGet-compatible algorithm. +#[must_use] +pub fn normalize_catalog_publisher(value: &str) -> String { + name_normalization::normalize_publisher(value) +} + const DEFAULT_MARKET: &str = "US"; const DEFAULT_MAX_RESULTS: usize = 50; const LIST_LOOKUP_MAX_RESULTS: usize = 500; @@ -8317,7 +8345,20 @@ fn extract_zip_member(bytes: &[u8], member_name: &str) -> Result> { Ok(output) } +/// Signature of the Windows Compression API buffer framing used by WinGet `.mszyml` files. +const COMPRESSION_API_SIGNATURE: [u8; 4] = [0x0a, 0x51, 0xe5, 0xc0]; +/// Size of the Compression API header: signature, 4 header bytes, u64 content size, u64 block size. +const COMPRESSION_API_HEADER_LEN: usize = 24; +/// MSZIP limits each block to 32 KiB of output and keeps a 32 KiB history window across blocks. +const MSZIP_BLOCK_SIZE: usize = 32 * 1024; +/// Upper bound on decoded version-data sidecars, including legacy unframed input. +const MAX_MSZYML_SIZE: usize = 64 * 1024 * 1024; + fn decompress_mszyml(bytes: &[u8]) -> Result { + if bytes.starts_with(&COMPRESSION_API_SIGNATURE) { + let output = decompress_compression_api_mszip(bytes)?; + return String::from_utf8(output).context("MSZIP payload is not valid UTF-8"); + } if let Ok(output) = decompress_mszyml_payload(if bytes.starts_with(b"CK") { &bytes[2..] } else { bytes }) { return Ok(output); } @@ -8331,10 +8372,80 @@ fn decompress_mszyml(bytes: &[u8]) -> Result { decompress_mszyml_payload(bytes).context("failed to decompress MSZIP payload") } +/// Decodes every `[u32 length]["CK" + deflate]` block of a Compression API MSZIP buffer. +/// +/// Each block may back-reference the previous block's output, so the history window is +/// replayed ahead of the block as an uncompressed deflate block. +fn decompress_compression_api_mszip(bytes: &[u8]) -> Result> { + let read_u64 = |offset: usize| -> Result { + let field: [u8; 8] = bytes + .get(offset..offset + 8) + .and_then(|slice| slice.try_into().ok()) + .context("truncated MSZIP header")?; + usize::try_from(u64::from_le_bytes(field)).context("MSZIP header size is too large") + }; + let expected_len = read_u64(8)?; + if expected_len > MAX_MSZYML_SIZE { + bail!("MSZIP payload exceeds the 64 MiB decoded size limit"); + } + let block_size = read_u64(16)?; + if block_size == 0 || block_size > MSZIP_BLOCK_SIZE { + bail!("unsupported MSZIP block size {block_size}"); + } + let mut output = Vec::with_capacity(expected_len.min(MSZIP_BLOCK_SIZE)); + let mut offset = COMPRESSION_API_HEADER_LEN; + while offset < bytes.len() { + let length_field: [u8; 4] = bytes + .get(offset..offset + 4) + .and_then(|slice| slice.try_into().ok()) + .context("truncated MSZIP block length")?; + let block_len = usize::try_from(u32::from_le_bytes(length_field)).context("MSZIP block length is too large")?; + offset += 4; + let end = offset.checked_add(block_len).context("MSZIP block length overflow")?; + let block = bytes.get(offset..end).context("truncated MSZIP block")?; + offset = end; + let deflate = block + .strip_prefix(b"CK") + .context("MSZIP block is missing its CK signature")?; + + let history = &output[output.len().saturating_sub(MSZIP_BLOCK_SIZE)..]; + let history_len = u16::try_from(history.len()).context("MSZIP history window is too large")?; + let mut stream = Vec::with_capacity(5 + history.len()); + stream.push(0x00); + stream.extend_from_slice(&history_len.to_le_bytes()); + stream.extend_from_slice(&(!history_len).to_le_bytes()); + stream.extend_from_slice(history); + + let mut decoded = Vec::with_capacity(history.len() + block_size); + flate2::read::DeflateDecoder::new(stream.as_slice().chain(deflate)) + .take((history.len() + block_size + 1) as u64) + .read_to_end(&mut decoded) + .context("failed to decompress MSZIP block")?; + let produced = decoded.len() - history.len(); + if produced == 0 || produced > block_size { + bail!("MSZIP block produced {produced} bytes; expected 1..={block_size}"); + } + if output.len() + produced > expected_len { + bail!("MSZIP payload exceeds its declared size of {expected_len} bytes"); + } + output.extend_from_slice(&decoded[history.len()..]); + } + if output.len() != expected_len { + bail!( + "MSZIP payload produced {} bytes; header declares {expected_len}", + output.len() + ); + } + Ok(output) +} + fn decompress_mszyml_payload(payload: &[u8]) -> Result { - let mut decoder = flate2::read::DeflateDecoder::new(payload); + let mut decoder = flate2::read::DeflateDecoder::new(payload).take((MAX_MSZYML_SIZE + 1) as u64); let mut output = String::new(); decoder.read_to_string(&mut output)?; + if output.len() > MAX_MSZYML_SIZE { + bail!("MSZIP payload exceeds the 64 MiB decoded size limit"); + } Ok(output) } @@ -13972,6 +14083,99 @@ Installers: assert_eq!(parsed.versions[0].version, "1.2.3"); } + /// Native Windows Compression API MSZIP output with three blocks whose later blocks + /// back-reference the previous block's history. + const NATIVE_MULTI_BLOCK_MSZIP_HEX: &[&str] = &[ + "0a51e5c018005202b03001000000000000800000000000009f010000434bedd4bb8d10410044419f2808819d9dcf6e40", + "08e19c30c85fa7f31e1221547bed3dabfefcfef8f5f3eff71f5ffb9673f58c9ebb67f6ac9edd737a9e9e37e76ac1d582", + "ab05570bae165c2db85a70b5e06ac1d582d182d182d182d182d182d182d182d182d182d182bb05770bee16dc2db85b70", + "b7e06ec1dd82bb05770b660b660b660b660b660b660b660b660b660b660b560b560b560b560b560b560b560b560b560b", + "560b760b760b760b760b760b760b760b760b760b760b4e0b4e0b4e0b4e0b4e0b4e0b4e0b4e0b4e0b4e0b9e163c2d785a", + "f0b4e069c1d382a7054f0b9e163c2d785bf0b6e06dc1db82b7056f0bde7f44821dec60073bd8c10e76b0831dec60073b", + "d8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76", + "b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec", + "60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8", + "c10e76b0831decfe87dd27c3000000434bedd42101000000c3a0bc37eb2f5f0442203bd9c94e76b2939dec64273bd9c9", + "4e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b293", + "9dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec6427", + "3bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e76b2939dec64273bd9c94e", + "76b2939dec64273bd9c94e76b2939decd60157000000434bedd6310d00000080a0fead2de1490906ec60073bd8991dec", + "60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8c10e76b0831dec60073bd8", + "c10e76b0831dec6007bb1bbb00", + ]; + + #[test] + fn decompresses_native_multi_block_mszyml() { + let hex: String = NATIVE_MULTI_BLOCK_MSZIP_HEX.concat(); + let bytes: Vec = (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("valid hex")) + .collect(); + let expected: String = (0..6000).map(|index| format!("pinget {:05}\n", index % 97)).collect(); + assert_eq!(expected.len(), 78_000); + + let output = decompress_mszyml(&bytes).expect("decompress multi-block payload"); + assert_eq!(output, expected); + + let truncated = &bytes[..bytes.len() - 10]; + assert!(decompress_mszyml(truncated).is_err(), "truncated stream must fail"); + assert!( + decompress_mszyml(&bytes[..COMPRESSION_API_HEADER_LEN - 4]).is_err(), + "truncated header must fail" + ); + + let mut wrong_size = bytes.clone(); + wrong_size[8] ^= 1; + assert!( + decompress_mszyml(&wrong_size).is_err(), + "declared size mismatch must fail" + ); + + let mut oversized_block = bytes.clone(); + oversized_block[16..24].copy_from_slice(&(64u64 * 1024).to_le_bytes()); + assert!( + decompress_mszyml(&oversized_block).is_err(), + "block size above 32 KiB must fail" + ); + + let mut missing_ck = bytes.clone(); + missing_ck[COMPRESSION_API_HEADER_LEN + 4] = b'X'; + assert!( + decompress_mszyml(&missing_ck).is_err(), + "missing CK signature must fail" + ); + + let mut bad_block_length = bytes; + bad_block_length[COMPRESSION_API_HEADER_LEN..COMPRESSION_API_HEADER_LEN + 4] + .copy_from_slice(&u32::MAX.to_le_bytes()); + assert!( + decompress_mszyml(&bad_block_length).is_err(), + "block length past end must fail" + ); + } + + #[test] + fn mszyml_size_limit_rejects_hostile_header_before_allocation() { + let mut header = [0u8; COMPRESSION_API_HEADER_LEN]; + header[..4].copy_from_slice(&COMPRESSION_API_SIGNATURE); + header[16..24].copy_from_slice(&(MSZIP_BLOCK_SIZE as u64).to_le_bytes()); + header[8..16].copy_from_slice(&(MAX_MSZYML_SIZE as u64).to_le_bytes()); + let boundary = decompress_mszyml(&header).expect_err("missing blocks"); + assert!( + !boundary.to_string().contains("64 MiB"), + "boundary accepted; absent output rejected" + ); + header[8..16].copy_from_slice(&((MAX_MSZYML_SIZE as u64) + 1).to_le_bytes()); + assert!( + decompress_mszyml(&header) + .expect_err("size limit") + .to_string() + .contains("64 MiB") + ); + header[8..16].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(decompress_mszyml(&header).is_err(), "hostile size must not allocate"); + } + #[test] fn decompresses_mszyml_payload_with_prefix() { let payload = "sV: 1.0.0\nvD:\n - v: 1.2.3\n rP: manifests/test.yaml\n s256H: ABCD\n"; diff --git a/rust/crates/pinget-publisher/Cargo.toml b/rust/crates/pinget-publisher/Cargo.toml new file mode 100644 index 0000000..06cef59 --- /dev/null +++ b/rust/crates/pinget-publisher/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "pinget-publisher" +version = "0.14.0" +edition = "2024" +description = "Native WinGet v2 preindexed source publisher for Pinget" +license = "MIT" +repository = "https://github.com/Devolutions/pinget" +exclude = ["tests/**"] + +[lints] +workspace = true + +[dependencies] +chrono = { version = "0.4.44", features = ["serde"] } +flate2 = "1.1.9" +fs2 = "0.4" +quick-xml = "0.38" +pinget-core = { version = "0.14.0", path = "../pinget-core" } +rusqlite = { version = "0.37", features = ["bundled"] } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.149" +serde_yaml = "0.9.34" +sha2 = "0.11.0" +thiserror = "2" +zip = "8.6.0" diff --git a/rust/crates/pinget-publisher/src/index-schema.sql b/rust/crates/pinget-publisher/src/index-schema.sql new file mode 100644 index 0000000..5ed0da9 --- /dev/null +++ b/rust/crates/pinget-publisher/src/index-schema.sql @@ -0,0 +1,53 @@ +CREATE TABLE metadata( + name TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE packages( + rowid INTEGER PRIMARY KEY, + id TEXT NOT NULL, + name TEXT NOT NULL, + moniker TEXT, + latest_version TEXT NOT NULL, + arp_min_version TEXT, + arp_max_version TEXT, + hash BLOB +); + +CREATE TABLE tags2(rowid INTEGER PRIMARY KEY, tag TEXT NOT NULL); +CREATE TABLE tags2_map( + tag INTEGER NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(tag, package) +) WITHOUT ROWID; +CREATE TABLE commands2(rowid INTEGER PRIMARY KEY, command TEXT NOT NULL); +CREATE TABLE commands2_map( + command INTEGER NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(command, package) +) WITHOUT ROWID; +CREATE TABLE pfns2( + pfn TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(pfn, package) +) WITHOUT ROWID; +CREATE TABLE productcodes2( + productcode TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(productcode, package) +) WITHOUT ROWID; +CREATE TABLE norm_names2( + norm_name TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(norm_name, package) +) WITHOUT ROWID; +CREATE TABLE norm_publishers2( + norm_publisher TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(norm_publisher, package) +) WITHOUT ROWID; +CREATE TABLE upgradecodes2( + upgradecode TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(upgradecode, package) +) WITHOUT ROWID; diff --git a/rust/crates/pinget-publisher/src/lib.rs b/rust/crates/pinget-publisher/src/lib.rs new file mode 100644 index 0000000..96c849d --- /dev/null +++ b/rust/crates/pinget-publisher/src/lib.rs @@ -0,0 +1,3931 @@ +//! Builds and publishes signed WinGet v2 static preindexed sources. +//! +//! The publisher reads standard manifest YAML without opening client source +//! state, writes immutable snapshots, and publishes only from those snapshots. +//! It enforces an explicit supported `ManifestVersion` set and structural +//! publishing invariants; it is not a full manifest-schema validator. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use chrono::Utc; +use quick_xml::Reader; +use quick_xml::events::Event; +use rusqlite::{Connection, OptionalExtension, params}; +use serde::ser::{SerializeMap, SerializeSeq, Serializer}; +use serde::{Deserialize, Serialize}; +use serde_yaml::{Mapping, Value}; +use sha2::{Digest, Sha256}; +use thiserror::Error; +use zip::ZipArchive; + +const SUPPORTED_MANIFEST_VERSIONS: &[&str] = &[ + "1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "1.6.0", "1.7.0", "1.8.0", "1.9.0", "1.10.0", +]; +const INDEX_SCHEMA: &str = include_str!("index-schema.sql"); +const STATE_SCHEMA: &str = include_str!("state-schema.sql"); +const MAX_MSZIP_SIZE: usize = 64 * 1024 * 1024; +const PUBLISHER_FORMAT_VERSION: u32 = 2; + +/// Errors returned by the repository publisher. +#[derive(Debug, Error)] +pub enum PublisherError { + /// An underlying filesystem operation failed. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + /// An underlying SQLite operation failed. + #[error("SQLite error: {0}")] + Sqlite(#[from] rusqlite::Error), + /// A manifest could not be parsed. + #[error("manifest YAML error in {path}: {source}")] + Yaml { + /// Manifest path. + path: PathBuf, + /// Parser error. + #[source] + source: serde_yaml::Error, + }, + /// A valid parsed YAML value could not be serialized. + #[error("manifest YAML serialization error: {0}")] + YamlSerialization(#[from] serde_yaml::Error), + /// State or ownership data could not be parsed. + #[error("publisher metadata error: {0}")] + Json(#[from] serde_json::Error), + /// Manifest input violated a publishing invariant. + #[error("{0}")] + Invalid(String), + /// A requested platform tool is unavailable or failed. + #[error("{0}")] + Tool(String), +} + +/// Result type used by publisher operations. +pub type Result = std::result::Result; + +/// Input locations for a new immutable repository snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildOptions { + /// Directory containing WinGet manifest YAML files. + pub input: PathBuf, + /// Publisher work directory holding state and immutable snapshots. + pub state: PathBuf, +} + +/// Summary of a completed build snapshot. +/// +/// The serialized form is the shared CLI JSON contract for `repository build`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct BuildReport { + /// Immutable snapshot identifier. + pub build_id: String, + /// Absolute snapshot directory under the state root. + pub snapshot_directory: PathBuf, + /// Number of merged hosted manifest artifacts. + #[serde(rename = "manifestCount")] + pub manifests: usize, + /// Number of package identifiers. + #[serde(rename = "packageCount")] + pub packages: usize, + /// Input files added or changed relative to the previous build (0 for the first build). + pub changed_files: usize, + /// Input files removed relative to the previous build. + pub removed_files: usize, + /// Number of immutable snapshot files (feed artifacts plus the index). + #[serde(skip)] + pub files: usize, +} + +/// Difference between current inputs and the most recent successful build. +/// +/// The serialized form is the shared CLI JSON contract for `repository diff`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DiffReport { + /// Most recent successful build, if any. + pub last_build_id: Option, + /// Whether the last successful build has a completed publication record. + #[serde(rename = "published")] + pub last_build_published: bool, + /// Relative paths added since the last build. + pub added: Vec, + /// Relative paths whose bytes changed since the last build. + pub changed: Vec, + /// Relative paths removed since the last build. + pub removed: Vec, +} + +/// Summary of publisher state without rescanning input manifests. +/// +/// The serialized form is the shared CLI JSON contract for `repository status`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct StatusReport { + /// Most recent successful build, if any. + pub last_build_id: Option, + /// Most recently published build, if any. + pub last_published_build_id: Option, + /// Number of builds recorded in state. + pub build_count: usize, + /// Creation time (UTC) of the most recent build. + pub last_build_timestamp: Option, + /// Most recent build details, if one exists. + #[serde(skip)] + pub last_build: Option, + /// Whether the most recent build has been published. + #[serde(skip)] + pub last_build_published: bool, +} + +/// Result of a successful snapshot or destination verification. +/// +/// The serialized form is the shared CLI JSON contract for `repository verify`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct VerifyReport { + /// Always true; failures are returned as errors. + pub valid: bool, + /// What was verified. + pub subject: String, + /// Number of verified artifacts. + pub artifact_count: usize, + /// Verified build, when known. + pub build_id: Option, +} + +/// Explicit packaging policy for a local publication. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Signing { + /// Sign with the supplied PFX and obtain its password from the named environment variable. + Pfx { path: PathBuf, password_env: String }, + /// Permit publishing an unsigned package for development only. + UnsignedDevelopment, +} + +/// Inputs required to package and publish an immutable snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublishOptions { + /// Publisher state directory. + pub state: PathBuf, + /// Snapshot to publish; `None` selects the latest successful snapshot. + pub build_id: Option, + /// Local static hosting directory. + pub destination: PathBuf, + /// Stable AppX package identity name. + pub name: String, + /// Publisher distinguished name used by the package and signing certificate. + pub publisher: String, + /// Four-component AppX package version. + pub version: String, + /// Signing policy. + pub signing: Signing, + /// Optional path to `makeappx.exe`; PATH and installed Windows SDKs are searched when omitted. + pub makeappx: Option, + /// Optional path to `signtool.exe`; PATH and installed Windows SDKs are searched when omitted. + pub signtool: Option, + /// Optional RFC 3161 timestamp URL used for signed publication. + pub timestamp_url: Option, +} + +/// Local publication result. +/// +/// The serialized form is the shared CLI JSON contract for `repository publish`: +/// the verification result of the published destination. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct PublishReport { + /// True only after the destination passed full verification. + pub valid: bool, + /// What was verified. + pub subject: String, + /// Number of published artifacts listed in the ownership marker. + pub artifact_count: usize, + /// Snapshot that was published. + pub build_id: String, + /// Destination directory. + #[serde(skip)] + pub destination: PathBuf, + /// Whether the result is unsigned and development-only. + #[serde(skip)] + pub unsigned_development: bool, + /// Whether this was an idempotent retry of an already-published build. + #[serde(skip)] + pub reused: bool, +} + +/// Tool and platform capabilities discovered by `doctor`. +/// +/// The serialized form is the shared CLI JSON contract for `repository doctor`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DoctorReport { + /// Informational host operating-system description. + pub operating_system: String, + /// Resolved `makeappx.exe`, if found. + #[serde(rename = "makeAppxPath")] + pub makeappx_path: Option, + /// Resolved `signtool.exe`, if found. + #[serde(rename = "signToolPath")] + pub signtool_path: Option, + /// Host can generate the native WinGet MSZIP stream. + #[serde(rename = "nativeMsZipAvailable")] + pub native_mszip_available: bool, + /// Whether this host supports signed MSIX publication. + #[serde(skip)] + pub signed_publish: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Inventory { + files: BTreeMap, + manifests: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SnapshotMetadata { + schema_version: u32, + #[serde(default)] + artifact_format_version: u32, + build_id: String, + created_utc: String, + manifest_count: usize, + package_count: usize, + index_hash: String, + #[serde(serialize_with = "serialize_inventory")] + inventory: BTreeMap, + #[serde(serialize_with = "serialize_inventory")] + artifacts: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ManifestInventory { + id: String, + name: String, + version: String, + moniker: Option, + latest_version: String, + manifest_path: String, + manifest_hash: String, + sidecar_path: String, + sidecar_hash: String, + tags: Vec, + commands: Vec, + pfns: Vec, + product_codes: Vec, + upgrade_codes: Vec, + normalized_names: Vec, + normalized_publishers: Vec, + arp_min_version: Option, + arp_max_version: Option, +} + +#[derive(Debug, Clone)] +struct ManifestDocument { + path: PathBuf, + kind: String, + id: String, + version: String, + locale: Option, + document: Value, +} + +type ManifestGroups = BTreeMap<(String, String), Vec>; + +#[derive(Debug, Clone)] +struct PackageManifest { + id: String, + name: String, + version: String, + moniker: Option, + locale_docs: Vec, +} + +#[derive(Debug, Clone)] +struct LocaleDocument { + locale: String, + document: Value, +} + +struct CanonicalYaml<'a>(&'a Value); + +impl Serialize for CanonicalYaml<'_> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + match self.0 { + Value::Null => serializer.serialize_unit(), + Value::Bool(value) => serializer.serialize_bool(*value), + Value::Number(value) => value.serialize(serializer), + Value::String(value) => serializer.serialize_str(value), + Value::Sequence(values) => { + let mut sequence = serializer.serialize_seq(Some(values.len()))?; + for value in values { + sequence.serialize_element(&CanonicalYaml(value))?; + } + sequence.end() + } + Value::Mapping(values) => { + let mut entries: Vec<_> = values.iter().collect(); + entries.sort_by(|(left, _), (right, _)| { + compare_utf16(left.as_str().unwrap_or_default(), right.as_str().unwrap_or_default()) + }); + let mut map = serializer.serialize_map(Some(entries.len()))?; + for (key, value) in entries { + let key = key + .as_str() + .ok_or_else(|| serde::ser::Error::custom("manifest JSON object keys must be strings"))?; + map.serialize_entry(key, &CanonicalYaml(value))?; + } + map.end() + } + Value::Tagged(_) => Err(serde::ser::Error::custom( + "YAML tags are not supported in WinGet manifests", + )), + } + } +} + +struct CanonicalInventory<'a>(&'a BTreeMap); + +impl Serialize for CanonicalInventory<'_> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let mut entries: Vec<_> = self.0.iter().collect(); + entries.sort_by(|(left, _), (right, _)| compare_utf16(left, right)); + let mut map = serializer.serialize_map(Some(entries.len()))?; + for (path, hash) in entries { + map.serialize_entry(path, hash)?; + } + map.end() + } +} + +struct WriterLock { + _file: File, +} + +/// Builds a complete, self-contained v2 index snapshot from manifest files. +/// +/// The input and state roots must not overlap. On Windows, version data is +/// compressed using the native MSZIP implementation used by WinGet. +/// +/// # Errors +/// +/// Returns an error for unsafe paths, unsupported manifest schema versions, +/// malformed or incomplete manifest sets, filesystem failures, or SQLite failures. +pub fn build(options: &BuildOptions) -> Result { + reject_reparse_components(&options.input)?; + let input = fs::canonicalize(&options.input)?; + if !input.is_dir() { + return Err(PublisherError::Invalid(format!( + "manifest input is not a directory: {}", + input.display() + ))); + } + let state_abs = canonicalize_future_path(&options.state)?; + ensure_disjoint_roots(&input, &state_abs)?; + fs::create_dir_all(&state_abs)?; + let state = fs::canonicalize(&state_abs)?; + ensure_disjoint_roots(&input, &state)?; + if state.join("publisher.db").exists() { + drop(open_state(&state, false)?); + } + let _lock = acquire_lock(&state)?; + + let (inventory, groups) = scan_manifests(&input)?; + if groups.is_empty() { + return Err(PublisherError::Invalid(format!( + "no supported manifest YAML files found under {}", + input.display() + ))); + } + let mut previous: Option> = None; + if state.join("publisher.db").is_file() { + let connection = open_state(&state, false)?; + let latest: Option = connection + .query_row("SELECT value FROM metadata WHERE name='lastBuildId'", [], |row| { + row.get(0) + }) + .optional()?; + if let Some(build_id) = latest { + let old_inventory: String = + connection.query_row("SELECT inventory FROM builds WHERE id=?1", [&build_id], |row| { + row.get(0) + })?; + let old: BTreeMap = serde_json::from_str(&old_inventory)?; + if old == inventory.files { + let snapshot = state.join("builds").join(&build_id); + verify_snapshot_path(&snapshot)?; + let metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; + validate_recorded_snapshot(&connection, &metadata)?; + return report_from_snapshot(&snapshot, &build_id, (0, 0)); + } + previous = Some(old); + } + } + let changes = previous.as_ref().map_or((0, 0), |old| { + let changed = inventory + .files + .iter() + .filter(|(path, hash)| old.get(*path) != Some(*hash)) + .count(); + let removed = old.keys().filter(|path| !inventory.files.contains_key(*path)).count(); + (changed, removed) + }); + let build_id = build_identifier(&inventory)?; + let snapshots = state.join("builds"); + reject_reparse_components(&snapshots)?; + fs::create_dir_all(&snapshots)?; + let snapshot = snapshots.join(&build_id); + if snapshot.exists() { + verify_snapshot_path(&snapshot)?; + let snapshot_data: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; + record_build(&state, &build_id, &snapshot_data)?; + return report_from_snapshot(&snapshot, &build_id, changes); + } + let staging = snapshots.join(format!(".staging-{build_id}")); + if staging.exists() { + remove_owned_staging(&staging)?; + } + fs::create_dir(&staging)?; + let result = (|| { + build_snapshot(&staging, groups, &build_id)?; + let snapshot_data = snapshot_metadata(&staging, &inventory, &build_id)?; + write_json(&staging.join("snapshot.json"), &snapshot_data)?; + Ok::<_, PublisherError>(snapshot_data) + })(); + let snapshot_data = match result { + Ok(snapshot_data) => snapshot_data, + Err(error) => { + let _ = fs::remove_dir_all(&staging); + return Err(error); + } + }; + if let Err(error) = fs::rename(&staging, &snapshot) { + let _ = fs::remove_dir_all(&staging); + return Err(error.into()); + } + record_build(&state, &build_id, &snapshot_data)?; + report_from_snapshot(&snapshot, &build_id, changes) +} + +/// Computes input changes relative to the most recent successful build. +/// +/// # Errors +/// +/// Returns an error if state is absent or unreadable, or if manifest inputs are invalid. +pub fn diff(options: &BuildOptions) -> Result { + reject_reparse_components(&options.input)?; + reject_reparse_components(&options.state)?; + let input = fs::canonicalize(&options.input)?; + let state = fs::canonicalize(&options.state)?; + ensure_disjoint_roots(&input, &state)?; + let connection = open_state(&state, false)?; + let previous: Option = connection + .query_row("SELECT value FROM metadata WHERE name='lastBuildId'", [], |row| { + row.get(0) + }) + .optional()?; + let (current, _) = scan_manifests(&input)?; + let Some(build_id) = previous else { + return Ok(DiffReport { + added: current.files.keys().cloned().collect(), + ..DiffReport::default() + }); + }; + let previous_inventory: String = + connection.query_row("SELECT inventory FROM builds WHERE id=?1", [&build_id], |row| { + row.get(0) + })?; + let previous_files: BTreeMap = serde_json::from_str(&previous_inventory)?; + let mut added = Vec::new(); + let mut changed = Vec::new(); + let mut removed = Vec::new(); + for (path, hash) in ¤t.files { + match previous_files.get(path) { + None => added.push(path.clone()), + Some(previous_hash) if previous_hash != hash => changed.push(path.clone()), + _ => {} + } + } + for path in previous_files.keys() { + if !current.files.contains_key(path) { + removed.push(path.clone()); + } + } + let published = is_published(&connection, &build_id)?; + Ok(DiffReport { + last_build_id: Some(build_id), + last_build_published: published, + added, + changed, + removed, + }) +} + +/// Reads publisher state without creating or changing files. +/// +/// # Errors +/// +/// Returns an error if the state directory is absent, incompatible, or unreadable. +pub fn status(state: impl AsRef) -> Result { + reject_reparse_components(state.as_ref())?; + let state = fs::canonicalize(state)?; + let connection = open_state(&state, false)?; + let latest: Option = connection + .query_row("SELECT value FROM metadata WHERE name='lastBuildId'", [], |row| { + row.get(0) + }) + .optional()?; + let last_published_build_id: Option = connection + .query_row( + "SELECT value FROM metadata WHERE name='lastPublishedBuildId'", + [], + |row| row.get(0), + ) + .optional()?; + let build_count: i64 = connection.query_row("SELECT COUNT(*) FROM builds", [], |row| row.get(0))?; + let build_count = usize::try_from(build_count) + .map_err(|_| PublisherError::Invalid("publisher build count is invalid".to_owned()))?; + let Some(build_id) = latest else { + return Ok(StatusReport { + last_published_build_id, + build_count, + ..StatusReport::default() + }); + }; + let (inventory, summary): (String, String) = connection.query_row( + "SELECT inventory, summary FROM builds WHERE id=?1", + [&build_id], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let summary: serde_json::Value = serde_json::from_str(&summary)?; + let report = BuildReport { + build_id: build_id.clone(), + snapshot_directory: display_path(&state.join("builds").join(&build_id)), + packages: json_count(&summary, "packageCount")?, + manifests: json_count(&summary, "manifestCount")?, + changed_files: 0, + removed_files: 0, + files: serde_json::from_str::>(&inventory)?.len(), + }; + Ok(StatusReport { + last_build_published: is_published(&connection, &build_id)?, + last_build_timestamp: summary["createdUtc"].as_str().map(str::to_owned), + last_build_id: Some(build_id), + last_published_build_id, + build_count, + last_build: Some(report), + }) +} + +/// Checks a snapshot or a previously published static directory. +/// +/// # Errors +/// +/// Returns an error if any inventory file is missing or differs from its stored hash. +pub fn verify(state: impl AsRef, build_id: Option<&str>, destination: Option<&Path>) -> Result { + if let Some(destination) = destination { + if build_id.is_some() { + return Err(PublisherError::Invalid( + "--build cannot be used with --destination".to_owned(), + )); + } + return verify_destination(destination); + } + reject_reparse_components(state.as_ref())?; + let state = fs::canonicalize(state)?; + let connection = open_state(&state, false)?; + let selected = match build_id { + Some(value) => value.to_owned(), + None => connection.query_row("SELECT value FROM metadata WHERE name='lastBuildId'", [], |row| { + row.get::<_, String>(0) + })?, + }; + validate_build_id(&selected)?; + let snapshot = state.join("builds").join(&selected); + verify_snapshot_path(&snapshot)?; + let metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; + validate_recorded_snapshot(&connection, &metadata)?; + Ok(VerifyReport { + valid: true, + subject: "unsigned build snapshot".to_owned(), + artifact_count: metadata.artifacts.len() + 1, + build_id: Some(selected), + }) +} + +/// Reports packaging tools available on the current host. +#[must_use] +pub fn doctor() -> DoctorReport { + doctor_with_tools(None, None) +} + +/// Reports publisher capabilities using optional explicit tool paths. +#[must_use] +pub fn doctor_with_tools(makeappx_path: Option<&Path>, signtool_path: Option<&Path>) -> DoctorReport { + let makeappx = find_tool("makeappx.exe", makeappx_path); + let signtool = find_tool("signtool.exe", signtool_path); + let powershell = find_tool("powershell.exe", None).is_some(); + DoctorReport { + operating_system: format!("{} {}", std::env::consts::OS, std::env::consts::ARCH), + signed_publish: cfg!(windows) && makeappx.is_some() && signtool.is_some() && powershell, + makeappx_path: makeappx.map(|path| display_path(&path)), + signtool_path: signtool.map(|path| display_path(&path)), + native_mszip_available: cfg!(windows), + } +} + +/// Verifies a published static source and, unless unsigned-development, its MSIX signature. +/// +/// # Errors +/// +/// Returns an error when the ownership inventory, MSIX contents, or signature is invalid. +pub fn verify_published_destination(destination: &Path, signtool_path: Option<&Path>) -> Result { + verify_destination_with_tool(destination, signtool_path) +} + +/// Packages, signs when requested, verifies and publishes one immutable snapshot. +/// +/// # Errors +/// +/// Returns an error for unsupported hosts, invalid package identity/version, +/// missing tools, failed signatures, unsafe destinations, or conflicting immutable files. +pub fn publish(options: &PublishOptions) -> Result { + if !cfg!(windows) { + return Err(PublisherError::Tool( + "MSIX packaging and signing are currently supported only on Windows".to_owned(), + )); + } + reject_reparse_components(&options.state)?; + reject_reparse_components(&options.destination)?; + validate_package_identity(&options.name, &options.publisher)?; + validate_msix_version(&options.version)?; + if let Some(url) = &options.timestamp_url + && (!matches!(options.signing, Signing::Pfx { .. }) + || !(url.starts_with("https://") || url.starts_with("http://")) + || url.chars().any(char::is_whitespace)) + { + return Err(PublisherError::Invalid( + "--timestamp-url requires signed publication and an HTTP(S) URL".to_owned(), + )); + } + let state = fs::canonicalize(&options.state)?; + let destination_abs = canonicalize_future_path(&options.destination)?; + ensure_disjoint_roots(&state, &destination_abs)?; + let connection = open_state(&state, false)?; + let _lock = acquire_lock(&state)?; + let build_id = match &options.build_id { + Some(value) => value.clone(), + None => connection.query_row("SELECT value FROM metadata WHERE name='lastBuildId'", [], |row| { + row.get::<_, String>(0) + })?, + }; + validate_build_id(&build_id)?; + let snapshot = state.join("builds").join(&build_id); + verify_snapshot_path(&snapshot)?; + let snapshot_metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; + validate_recorded_snapshot(&connection, &snapshot_metadata)?; + + read_owner_marker(&destination_abs.join(".pinget-publisher.json"))?; + let _destination_lock = acquire_destination_lock(&destination_abs)?; + fs::create_dir_all(&destination_abs)?; + let destination = fs::canonicalize(&destination_abs)?; + ensure_disjoint_roots(&state, &destination)?; + ensure_owned_destination(&destination)?; + let marker_path = destination.join(".pinget-publisher.json"); + let prior = read_owner_marker(&marker_path)?; + let unsigned_development = matches!(options.signing, Signing::UnsignedDevelopment); + if let Some(owner) = prior.as_ref() { + let verified = verify_destination_with_tool(&destination, options.signtool.as_deref())?; + if owner.package_name != options.name || owner.publisher != options.publisher { + return Err(PublisherError::Invalid( + "destination is owned by a different package identity".to_owned(), + )); + } + if owner.build_id == build_id && owner.package_version == options.version { + if owner.unsigned_development != unsigned_development { + return Err(PublisherError::Invalid( + "the package version has already been published with a different signing mode".to_owned(), + )); + } + mark_published(&state, &build_id)?; + return Ok(PublishReport { + valid: true, + subject: verified.subject, + artifact_count: verified.artifact_count, + build_id, + destination: display_path(&destination), + unsigned_development: owner.unsigned_development, + reused: true, + }); + } + match compare_quad_version(&options.version, &owner.package_version)? { + std::cmp::Ordering::Less => { + return Err(PublisherError::Invalid(format!( + "package version {} must be greater than previously published version {}", + options.version, owner.package_version + ))); + } + std::cmp::Ordering::Equal => { + return Err(PublisherError::Invalid( + "the package version has already been used for different build content".to_owned(), + )); + } + std::cmp::Ordering::Greater => {} + } + } + let current_inventory = published_inventory(&destination)?; + if let Some(owner) = prior.as_ref() { + for (path, expected) in &owner.artifacts { + if current_inventory.get(path) != Some(expected) { + return Err(PublisherError::Invalid(format!( + "owned publication file is missing or modified: {path}" + ))); + } + } + for (path, actual) in ¤t_inventory { + if owner.artifacts.get(path) != Some(actual) && snapshot_metadata.artifacts.get(path) != Some(actual) { + return Err(PublisherError::Invalid(format!( + "destination contains an untracked or modified file: {path}" + ))); + } + } + } + let temporary = destination.join(format!(".publish-{build_id}.staging")); + if temporary.exists() { + remove_owned_staging(&temporary)?; + } + fs::create_dir(&temporary)?; + let result = package_and_publish( + &snapshot, + &temporary, + &destination, + options, + &build_id, + unsigned_development, + prior, + ); + if result.is_err() { + let _ = fs::remove_dir_all(&temporary); + } + result +} + +fn build_snapshot( + staging: &Path, + groups: BTreeMap<(String, String), Vec>, + build_id: &str, +) -> Result<()> { + let feed = staging.join("feed"); + fs::create_dir_all(&feed)?; + fs::create_dir_all(staging.join("Public"))?; + let mut db = Connection::open(staging.join("Public").join("index.db"))?; + db.execute_batch(INDEX_SCHEMA)?; + db.execute_batch("PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL;")?; + db.execute( + "INSERT INTO metadata(name,value) VALUES('majorVersion','2'),('minorVersion','0'),('databaseIdentifier',?1),('lastwritetime',?2)", + params![format!("{{{}}}", uuid_string(build_id)), Utc::now().timestamp().to_string()], + )?; + let transaction = db.transaction()?; + let mut by_package: BTreeMap> = BTreeMap::new(); + let mut package_spellings = BTreeMap::::new(); + for ((id, _), documents) in groups { + let package = merge_documents(&id, documents)?; + let key = package.id.to_ascii_lowercase(); + if let Some(previous) = package_spellings.get(&key) + && previous != &package.id + { + return Err(PublisherError::Invalid(format!( + "PackageIdentifier casing is inconsistent across manifests: {previous} and {}", + package.id + ))); + } + package_spellings + .entry(key.clone()) + .or_insert_with(|| package.id.clone()); + by_package.entry(key).or_default().push(package); + } + for (row_index, (_, mut versions)) in by_package.into_iter().enumerate() { + versions.sort_by(|left, right| compare_winget_versions(&left.version, &right.version)); + if versions + .windows(2) + .any(|pair| compare_winget_versions(&pair[0].version, &pair[1].version) == std::cmp::Ordering::Equal) + { + return Err(PublisherError::Invalid(format!( + "duplicate package version {} for {}", + versions[0].version, versions[0].id + ))); + } + let latest = versions + .last() + .ok_or_else(|| PublisherError::Invalid("empty package version group".to_owned()))? + .clone(); + let mut version_records = Vec::new(); + for package in &versions { + let mut locale_docs = package.locale_docs.clone(); + locale_docs.sort_by_key(|left| left.locale.to_ascii_lowercase()); + for locale in locale_docs { + let locale_package = PackageManifest { + id: package.id.clone(), + name: package.name.clone(), + version: package.version.clone(), + moniker: package.moniker.clone(), + locale_docs: vec![locale.clone()], + }; + let locale_fields = catalog_fields_all(std::slice::from_ref(&locale_package))?; + let manifest_bytes = serialize_merged(&locale.document)?; + let manifest_hash = sha256_hex(&manifest_bytes); + let manifest_relative = format!( + "manifests/{}/{}/{}/{}.yaml", + safe_component(&package.id)?, + safe_component(&package.version)?, + safe_component(&locale.locale)?, + &manifest_hash[..16] + ); + write_relative(&feed, &manifest_relative, &manifest_bytes)?; + version_records.push(( + package.version.clone(), + locale.locale, + manifest_relative, + manifest_hash, + locale_fields.7, + locale_fields.8, + )); + } + } + version_records + .sort_by(|left, right| compare_winget_versions(&right.0, &left.0).then_with(|| left.1.cmp(&right.1))); + let mut version_yaml = String::from("sV: \"1.0\"\nvD:\n"); + for (version, _, path, hash, min_version, max_version) in &version_records { + version_yaml.push_str(&format!(" - v: {}\n", json_quote(version)?,)); + if let Some(min_version) = min_version { + version_yaml.push_str(&format!(" aMiV: {}\n", json_quote(min_version)?)); + } + if let Some(max_version) = max_version { + version_yaml.push_str(&format!(" aMaV: {}\n", json_quote(max_version)?)); + } + version_yaml.push_str(&format!( + " rP: {}\n s256H: {}\n", + json_quote(path)?, + json_quote(hash)? + )); + } + let sidecar_bytes = compress_mszip(version_yaml.as_bytes())?; + let sidecar_hash = sha256_hex(&sidecar_bytes); + let sidecar_relative = format!( + "packages/{}/{}/versionData.mszyml", + safe_component(&latest.id)?, + &sidecar_hash[..8] + ); + write_relative(&feed, &sidecar_relative, &sidecar_bytes)?; + let fields = catalog_fields_all(&versions)?; + let row_id = + i64::try_from(row_index + 1).map_err(|_| PublisherError::Invalid("too many package rows".to_owned()))?; + transaction.execute( + "INSERT INTO packages(rowid,id,name,moniker,latest_version,arp_min_version,arp_max_version,hash) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)", + params![ + row_id, + latest.id, + latest.name, + latest.moniker, + latest.version, + fields.7, + fields.8, + Sha256::digest(&sidecar_bytes).as_slice() + ], + )?; + insert_terms(&transaction, "tags2", "tag", "tags2_map", "tag", row_id, &fields.0)?; + insert_terms( + &transaction, + "commands2", + "command", + "commands2_map", + "command", + row_id, + &fields.1, + )?; + insert_correlations(&transaction, "pfns2", "pfn", row_id, &fields.2)?; + insert_correlations(&transaction, "productcodes2", "productcode", row_id, &fields.3)?; + insert_correlations(&transaction, "upgradecodes2", "upgradecode", row_id, &fields.4)?; + insert_correlations(&transaction, "norm_names2", "norm_name", row_id, &fields.5)?; + insert_correlations(&transaction, "norm_publishers2", "norm_publisher", row_id, &fields.6)?; + } + transaction.commit()?; + drop(db); + let _ = build_id; + Ok(()) +} + +fn scan_manifests(root: &Path) -> Result<(Inventory, ManifestGroups)> { + let mut files = BTreeMap::new(); + let mut case_paths = BTreeSet::new(); + let mut documents = Vec::new(); + scan_directory(root, root, &mut files, &mut case_paths, &mut documents)?; + let mut grouped: BTreeMap<(String, String), Vec> = BTreeMap::new(); + let mut identifiers: BTreeMap = BTreeMap::new(); + for document in documents { + let spelling = identifiers + .entry(document.id.to_ascii_lowercase()) + .or_insert_with(|| document.id.clone()); + if spelling != &document.id { + return Err(PublisherError::Invalid(format!( + "case-conflicting package identifiers: {spelling} and {}", + document.id + ))); + } + grouped + .entry((document.id.clone(), document.version.clone())) + .or_default() + .push(document); + } + let mut manifests = Vec::with_capacity(grouped.len()); + for ((_, _), docs) in &grouped { + let first = docs + .first() + .ok_or_else(|| PublisherError::Invalid("empty manifest group".to_owned()))?; + let package = merge_documents(&first.id, docs.clone())?; + let fields = catalog_fields_all(std::slice::from_ref(&package))?; + manifests.push(ManifestInventory { + id: package.id.clone(), + name: package.name.clone(), + version: package.version.clone(), + latest_version: package.version.clone(), + moniker: package.moniker.clone(), + manifest_path: String::new(), + manifest_hash: String::new(), + sidecar_path: String::new(), + sidecar_hash: String::new(), + tags: fields.0, + commands: fields.1, + pfns: fields.2, + product_codes: fields.3, + upgrade_codes: fields.4, + normalized_names: fields.5, + normalized_publishers: fields.6, + arp_min_version: fields.7, + arp_max_version: fields.8, + }); + } + let inventory = Inventory { files, manifests }; + Ok((inventory, grouped)) +} + +fn scan_directory( + root: &Path, + directory: &Path, + files: &mut BTreeMap, + case_paths: &mut BTreeSet, + documents: &mut Vec, +) -> Result<()> { + let mut entries = fs::read_dir(directory)?.collect::>>()?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let metadata = fs::symlink_metadata(&path)?; + if is_reparse_point(&metadata) { + return Err(PublisherError::Invalid(format!( + "symbolic links and reparse points are not allowed in manifest input: {}", + path.display() + ))); + } + if metadata.is_dir() { + scan_directory(root, &path, files, case_paths, documents)?; + continue; + } + let extension = path.extension().and_then(|v| v.to_str()).unwrap_or_default(); + if !extension.eq_ignore_ascii_case("yaml") && !extension.eq_ignore_ascii_case("yml") { + continue; + } + let relative = path + .strip_prefix(root) + .map_err(|_| PublisherError::Invalid(format!("manifest escaped input root: {}", path.display())))?; + validate_relative_path(relative)?; + let path_key = relative + .to_str() + .ok_or_else(|| PublisherError::Invalid(format!("manifest path is not valid Unicode: {}", path.display())))? + .replace('\\', "/"); + if !case_paths.insert(path_key.to_lowercase()) { + return Err(PublisherError::Invalid(format!( + "case-colliding manifest path: {path_key}" + ))); + } + let bytes = fs::read(&path)?; + if bytes.len() > 8 * 1024 * 1024 { + return Err(PublisherError::Invalid(format!( + "manifest exceeds the 8 MiB limit: {path_key}" + ))); + } + files.insert(path_key, sha256_hex(&bytes)); + let deserializer = serde_yaml::Deserializer::from_slice(&bytes); + let mut any_document = false; + for document in deserializer { + if any_document { + return Err(PublisherError::Invalid(format!( + "expected one YAML document in {}", + path.display() + ))); + } + let value = serde::de::DeserializeSeed::deserialize(ManifestValue, document).map_err(|source| { + PublisherError::Yaml { + path: path.clone(), + source, + } + })?; + validate_yaml_value(&value, 0)?; + if value.is_null() { + continue; + } + any_document = true; + let mapping = value.as_mapping().ok_or_else(|| { + PublisherError::Invalid(format!("manifest document in {} is not a YAML mapping", path.display())) + })?; + let manifest_type = required_string(mapping, "ManifestType", &path)?; + let manifest_version = required_string(mapping, "ManifestVersion", &path)?; + if !SUPPORTED_MANIFEST_VERSIONS.contains(&manifest_version.as_str()) { + return Err(PublisherError::Invalid(format!( + "unsupported ManifestVersion {manifest_version} in {}; supported versions: {}", + path.display(), + SUPPORTED_MANIFEST_VERSIONS.join(", ") + ))); + } + let kind = manifest_type.clone(); + if !["singleton", "version", "installer", "defaultLocale", "locale"].contains(&kind.as_str()) { + return Err(PublisherError::Invalid(format!( + "unsupported ManifestType {manifest_type} in {}", + path.display() + ))); + } + let id = required_string(mapping, "PackageIdentifier", &path)?; + validate_package_identifier(&id)?; + let version = required_string(mapping, "PackageVersion", &path)?; + validate_package_version(&version)?; + let locale = optional_string(mapping, "PackageLocale"); + if matches!(kind.as_str(), "singleton" | "defaultLocale" | "locale") { + validate_locale( + locale.as_deref().ok_or_else(|| { + PublisherError::Invalid(format!("missing PackageLocale in {}", path.display())) + })?, + )?; + } + documents.push(ManifestDocument { + path: path.clone(), + kind: kind.to_ascii_lowercase(), + id, + version, + locale, + document: value, + }); + } + if !any_document { + return Err(PublisherError::Invalid(format!( + "manifest file is empty: {}", + path.display() + ))); + } + } + Ok(()) +} + +struct ManifestValue; + +struct ManifestString; + +impl<'de> serde::de::DeserializeSeed<'de> for ManifestString { + type Value = Value; + + fn deserialize>(self, deserializer: D) -> std::result::Result { + deserializer.deserialize_string(self) + } +} + +impl<'de> serde::de::Visitor<'de> for ManifestString { + type Value = Value; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a string-valued manifest field") + } + + fn visit_str(self, value: &str) -> std::result::Result { + Ok(Value::String(value.to_owned())) + } + + fn visit_map>(self, access: M) -> std::result::Result { + serde::de::Visitor::visit_map(ManifestValue, access) + } + + fn visit_seq>(self, mut access: S) -> std::result::Result { + let mut items = Vec::new(); + while let Some(value) = access.next_element_seed(ManifestString)? { + items.push(value); + } + Ok(Value::Sequence(items)) + } +} + +impl<'de> serde::de::DeserializeSeed<'de> for ManifestValue { + type Value = Value; + + fn deserialize>(self, deserializer: D) -> std::result::Result { + deserializer.deserialize_any(self) + } +} + +impl<'de> serde::de::Visitor<'de> for ManifestValue { + type Value = Value; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a manifest YAML value") + } + + fn visit_map>(self, mut access: M) -> std::result::Result { + let mut map = Mapping::new(); + while let Some(key) = access.next_key::()? { + let key = key + .as_str() + .ok_or_else(|| serde::de::Error::custom("manifest mapping keys must be strings"))? + .to_owned(); + let value = if matches!( + key.as_str(), + "Commands" + | "Tags" + | "FileExtensions" + | "Protocols" + | "Capabilities" + | "RestrictedCapabilities" + | "InstallModes" + | "Platform" + | "WindowsFeatures" + | "WindowsLibraries" + | "ExternalDependencies" + | "AllowedMarkets" + | "ExcludedMarkets" + | "UnsupportedOSArchitectures" + | "UnsupportedArguments" + ) { + Value::Sequence( + access + .next_value::>()? + .into_iter() + .map(Value::String) + .collect(), + ) + } else if is_manifest_string_field(&key) { + // deserialize_string asks YAML for its original scalar text, not its inferred number. + access.next_value_seed(ManifestString)? + } else { + access.next_value_seed(ManifestValue)? + }; + if map.insert(Value::String(key.clone()), value).is_some() { + return Err(serde::de::Error::custom(format!("duplicate manifest key {key}"))); + } + } + Ok(Value::Mapping(map)) + } + + fn visit_seq>(self, mut access: S) -> std::result::Result { + let mut items = Vec::new(); + while let Some(value) = access.next_element_seed(ManifestValue)? { + items.push(value); + } + Ok(Value::Sequence(items)) + } + + fn visit_str(self, value: &str) -> std::result::Result { + Ok(Value::String(value.to_owned())) + } + + fn visit_bool(self, value: bool) -> std::result::Result { + Ok(Value::Bool(value)) + } + + fn visit_i64(self, value: i64) -> std::result::Result { + Ok(Value::Number(value.into())) + } + + fn visit_u64(self, value: u64) -> std::result::Result { + Ok(Value::Number(value.into())) + } + + fn visit_f64(self, value: f64) -> std::result::Result { + if !value.is_finite() { + return Err(E::custom( + "non-finite numbers cannot be represented in merged manifest JSON", + )); + } + Ok(Value::Number(value.into())) + } + + fn visit_unit(self) -> std::result::Result { + Ok(Value::Null) + } +} + +fn is_manifest_string_field(key: &str) -> bool { + key.ends_with("Sha256") + || key.ends_with("Url") + || matches!( + key, + "PackageIdentifier" + | "PackageVersion" + | "ManifestVersion" + | "ManifestType" + | "DefaultLocale" + | "PackageLocale" + | "PackageName" + | "Publisher" + | "Moniker" + | "DisplayName" + | "DisplayVersion" + | "MinimumVersion" + | "MinimumOSVersion" + | "PackageFamilyName" + | "ProductCode" + | "UpgradeCode" + | "InstallerUrl" + | "InstallerLocale" + | "InstallerType" + | "NestedInstallerType" + | "Architecture" + | "Scope" + | "License" + | "ShortDescription" + | "Description" + | "Copyright" + | "PublisherUrl" + | "PublisherSupportUrl" + | "PrivacyUrl" + | "Author" + | "PackageUrl" + | "LicenseUrl" + | "CopyrightUrl" + | "ReleaseNotes" + | "ReleaseNotesUrl" + | "PurchaseUrl" + | "InstallationNotes" + | "InstallLocation" + | "RelativeFilePath" + | "PortableCommandAlias" + | "IconUrl" + | "IconFileType" + | "IconResolution" + | "IconTheme" + | "DocumentLabel" + | "DocumentUrl" + | "Silent" + | "SilentWithProgress" + | "Interactive" + | "Log" + | "Upgrade" + | "Custom" + | "Repair" + | "FileType" + | "ReleaseDate" + | "Commands" + | "Tags" + | "FileExtensions" + | "Protocols" + | "Capabilities" + | "RestrictedCapabilities" + | "InstallModes" + | "Platform" + | "UpgradeBehavior" + | "AuthenticationType" + | "Argument" + | "Command" + | "Uninstall" + | "Update" + | "Channel" + | "InvocationParameter" + | "ReturnResponse" + | "ElevationRequirement" + | "RepairBehavior" + | "InstallerIdentifier" + | "ProductId" + | "StoreProductId" + ) +} + +fn merge_documents(id: &str, documents: Vec) -> Result { + let mut singleton: Option = None; + let mut version: Option = None; + let mut default_locale: Option = None; + let mut locales: Vec<(String, Value)> = Vec::new(); + let mut installer: Option = None; + let mut seen_locale = BTreeSet::new(); + for document in documents { + if !document.id.eq_ignore_ascii_case(id) { + return Err(PublisherError::Invalid(format!( + "inconsistent PackageIdentifier {} in {}", + document.id, + document.path.display() + ))); + } + match document.kind.as_str() { + "singleton" => replace_once(&mut singleton, document.document, "singleton manifest")?, + "version" => replace_once(&mut version, document.document, "version manifest")?, + "installer" => replace_once(&mut installer, document.document, "installer manifest")?, + "defaultlocale" => replace_once(&mut default_locale, document.document, "default-locale manifest")?, + "locale" => { + let locale = document.locale.ok_or_else(|| { + PublisherError::Invalid(format!( + "locale manifest lacks PackageLocale: {}", + document.path.display() + )) + })?; + if !seen_locale.insert(locale.to_ascii_lowercase()) { + return Err(PublisherError::Invalid(format!("duplicate locale {locale} for {id}"))); + } + locales.push((locale, document.document)); + } + _ => return Err(PublisherError::Invalid("unsupported manifest type".to_owned())), + } + } + if let Some(singleton) = singleton { + if version.is_some() || default_locale.is_some() || installer.is_some() || !locales.is_empty() { + return Err(PublisherError::Invalid(format!( + "singleton and multi-file manifests are mixed for {id}" + ))); + } + let mapping = as_mapping(&singleton)?; + let locale = required_string(mapping, "PackageLocale", Path::new(""))?; + validate_locale(&locale)?; + if map_get(mapping, "Installers") + .and_then(Value::as_sequence) + .is_none_or(Vec::is_empty) + { + return Err(PublisherError::Invalid(format!("no installers found for {id}"))); + } + return Ok(PackageManifest { + id: required_string(mapping, "PackageIdentifier", Path::new(""))?, + name: required_string(mapping, "PackageName", Path::new(""))?, + version: required_string(mapping, "PackageVersion", Path::new(""))?, + moniker: optional_string(mapping, "Moniker"), + locale_docs: vec![LocaleDocument { + locale, + document: canonicalize_value(singleton), + }], + }); + } + let version = version.ok_or_else(|| PublisherError::Invalid(format!("missing version manifest for {id}")))?; + let version_map = as_mapping(&version)?.clone(); + let default_locale = + default_locale.ok_or_else(|| PublisherError::Invalid(format!("missing defaultLocale manifest for {id}")))?; + let default_map = as_mapping(&default_locale)?.clone(); + let version_id = required_string(&version_map, "PackageIdentifier", Path::new(""))?; + let version_number = required_string(&version_map, "PackageVersion", Path::new(""))?; + if !version_id.eq_ignore_ascii_case(id) { + return Err(PublisherError::Invalid(format!( + "inconsistent PackageIdentifier in {id}" + ))); + } + let installer_map = installer.as_ref().map(as_mapping).transpose()?; + let installer_map = installer_map + .ok_or_else(|| PublisherError::Invalid(format!("missing installer manifest for {id} {version_number}")))?; + let doc_id = required_string(installer_map, "PackageIdentifier", Path::new(""))?; + let doc_version = required_string(installer_map, "PackageVersion", Path::new(""))?; + if !doc_id.eq_ignore_ascii_case(id) || doc_version != version_number { + return Err(PublisherError::Invalid(format!( + "inconsistent installer manifest for {id} {version_number}" + ))); + } + let installer_entries = map_get(installer_map, "Installers") + .and_then(Value::as_sequence) + .cloned() + .unwrap_or_default(); + if installer_entries.is_empty() { + return Err(PublisherError::Invalid(format!( + "no installers found for {id} {version_number}" + ))); + } + let default_locale_name = required_string(&default_map, "PackageLocale", Path::new(""))?; + validate_locale(&default_locale_name)?; + let has_default_locale = locales + .iter() + .any(|(locale, _)| locale.eq_ignore_ascii_case(&default_locale_name)); + if has_default_locale { + return Err(PublisherError::Invalid(format!( + "duplicate default locale {default_locale_name} for {id}" + ))); + } + let declared_default = required_string(&version_map, "DefaultLocale", Path::new(""))?; + if !declared_default.eq_ignore_ascii_case(&default_locale_name) { + return Err(PublisherError::Invalid(format!( + "DefaultLocale {declared_default} does not match {default_locale_name} for {id}" + ))); + } + locales.sort_by(|left, right| compare_utf16(&left.0, &right.0)); + let mut localizations = Vec::with_capacity(locales.len()); + for (locale, locale_value) in locales { + validate_locale(&locale)?; + let mut locale_map = as_mapping(&locale_value)?.clone(); + for key in ["ManifestType", "ManifestVersion", "PackageIdentifier", "PackageVersion"] { + locale_map.remove(Value::String(key.to_owned())); + } + localizations.push(Value::Mapping(locale_map)); + } + let mut merged = version_map.clone(); + merge_yaml_mapping(&mut merged, installer_map); + merge_yaml_mapping(&mut merged, &default_map); + merged.insert( + Value::String("Installers".to_owned()), + Value::Sequence(installer_entries), + ); + if !localizations.is_empty() { + merged.insert(Value::String("Localization".to_owned()), Value::Sequence(localizations)); + } + set_string(&mut merged, "ManifestType", "singleton"); + let locale_docs = vec![LocaleDocument { + locale: default_locale_name, + document: canonicalize_value(Value::Mapping(merged)), + }]; + let name = required_string(&default_map, "PackageName", Path::new(""))?; + let moniker = optional_string(&default_map, "Moniker").or_else(|| optional_string(&version_map, "Moniker")); + Ok(PackageManifest { + id: version_id, + name, + version: version_number, + moniker, + locale_docs, + }) +} + +fn serialize_merged(document: &Value) -> Result> { + let mut bytes = serde_json::to_vec(&CanonicalYaml(document))?; + bytes.push(b'\n'); + Ok(bytes) +} + +type CatalogFields = ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Option, + Option, +); + +fn catalog_fields_all(packages: &[PackageManifest]) -> Result { + let mut tags = BTreeSet::new(); + let mut commands = BTreeSet::new(); + let mut pfns = BTreeSet::new(); + let mut product_codes = BTreeSet::new(); + let mut upgrade_codes = BTreeSet::new(); + let mut normalized_names = BTreeSet::new(); + let mut normalized_publishers = BTreeSet::new(); + let mut arp_versions = BTreeSet::new(); + for document in packages.iter().flat_map(|package| &package.locale_docs) { + let map = as_mapping(&document.document)?; + extend_strings(&mut tags, map, "Tags"); + extend_strings(&mut commands, map, "Commands"); + let mut scopes = vec![map]; + for key in ["Localization", "Installers"] { + if let Some(values) = map_get(map, key).and_then(Value::as_sequence) { + scopes.extend(values.iter().filter_map(Value::as_mapping)); + } + } + for nested in scopes { + extend_strings(&mut tags, nested, "Tags"); + extend_strings(&mut commands, nested, "Commands"); + extend_strings(&mut pfns, nested, "PackageFamilyName"); + extend_strings(&mut product_codes, nested, "ProductCode"); + extend_strings(&mut upgrade_codes, nested, "UpgradeCode"); + if let Some(value) = optional_string(nested, "PackageName") { + let normalized = pinget_core::normalize_catalog_name(&value); + if !normalized.name.is_empty() { + normalized_names.insert(normalized.name); + } + } + if let Some(value) = optional_string(nested, "Publisher") { + let normalized = pinget_core::normalize_catalog_publisher(&value); + if !normalized.is_empty() { + normalized_publishers.insert(normalized); + } + } + if let Some(entries) = map_get(nested, "AppsAndFeaturesEntries").and_then(Value::as_sequence) { + for entry in entries { + if let Some(entry_map) = entry.as_mapping() { + if let Some(value) = optional_string(entry_map, "DisplayName") { + let normalized = pinget_core::normalize_catalog_name(&value); + if !normalized.name.is_empty() { + normalized_names.insert(normalized.name); + } + } + if let Some(value) = optional_string(entry_map, "Publisher") { + let normalized = pinget_core::normalize_catalog_publisher(&value); + if !normalized.is_empty() { + normalized_publishers.insert(normalized); + } + } + if let Some(value) = + optional_string(entry_map, "DisplayVersion").filter(|value| !value.trim().is_empty()) + { + arp_versions.insert(value); + } + extend_strings(&mut product_codes, entry_map, "ProductCode"); + extend_strings(&mut upgrade_codes, entry_map, "UpgradeCode"); + } + } + } + if let Some(installers) = map_get(nested, "Installers").and_then(Value::as_sequence) { + for installer in installers { + if let Some(installer_map) = installer.as_mapping() { + extend_strings(&mut pfns, installer_map, "PackageFamilyName"); + extend_strings(&mut product_codes, installer_map, "ProductCode"); + extend_strings(&mut upgrade_codes, installer_map, "UpgradeCode"); + } + } + } + } + } + let mut versions: Vec = arp_versions.into_iter().collect(); + versions.sort_by(|left, right| compare_winget_versions(left, right)); + let min = versions.first().cloned(); + let max = versions.last().cloned(); + Ok(( + canonical_terms(tags), + canonical_terms(commands), + canonical_terms(pfns), + canonical_terms(product_codes), + canonical_terms(upgrade_codes), + canonical_terms(normalized_names), + canonical_terms(normalized_publishers), + min, + max, + )) +} + +fn canonical_terms(terms: BTreeSet) -> Vec { + let mut unique: BTreeMap = BTreeMap::new(); + for term in terms.into_iter().filter(|term| !term.trim().is_empty()) { + let key = term.to_ascii_lowercase(); + let current = unique.entry(key).or_insert_with(|| term.clone()); + if compare_utf16(&term, current).is_lt() { + *current = term; + } + } + let mut terms: Vec<_> = unique.into_values().collect(); + terms.sort_by(|left, right| compare_utf16(left, right)); + terms +} + +fn insert_terms( + connection: &Connection, + table: &str, + value_column: &str, + map_table: &str, + map_column: &str, + package_row: i64, + terms: &[String], +) -> Result<()> { + for term in terms { + let query = format!( + "SELECT rowid,{value_column} FROM {table} WHERE {value_column}=?1 COLLATE NOCASE ORDER BY rowid LIMIT 1" + ); + let existing: Option<(i64, String)> = connection + .query_row(&query, [term], |row| Ok((row.get(0)?, row.get(1)?))) + .optional()?; + let value_row = if let Some((value_row, spelling)) = existing { + if compare_utf16(term, &spelling).is_lt() { + connection.execute( + &format!("UPDATE {table} SET {value_column}=?1 WHERE rowid=?2"), + params![term, value_row], + )?; + } + value_row + } else { + connection.execute(&format!("INSERT INTO {table}({value_column}) VALUES(?1)"), [term])?; + connection.last_insert_rowid() + }; + let map_sql = format!("INSERT OR IGNORE INTO {map_table}({map_column},package) VALUES(?1,?2)"); + connection.execute(&map_sql, params![value_row, package_row])?; + } + Ok(()) +} + +fn insert_correlations( + connection: &Connection, + table: &str, + value_column: &str, + package_row: i64, + values: &[String], +) -> Result<()> { + let sql = format!("INSERT OR IGNORE INTO {table}({value_column},package) VALUES(?1,?2)"); + for value in values { + connection.execute(&sql, params![value, package_row])?; + } + Ok(()) +} + +fn snapshot_metadata(staging: &Path, inventory: &Inventory, build_id: &str) -> Result { + let artifacts = hash_tree(&staging.join("feed"))?; + let index_path = staging.join("Public").join("index.db"); + let index_hash = sha256_hex(&fs::read(&index_path)?); + let db = Connection::open_with_flags(index_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let package_count: i64 = db.query_row("SELECT COUNT(*) FROM packages", [], |row| row.get(0))?; + let manifest_count = artifacts.keys().filter(|path| path.starts_with("manifests/")).count(); + Ok(SnapshotMetadata { + schema_version: 1, + artifact_format_version: PUBLISHER_FORMAT_VERSION, + build_id: build_id.to_owned(), + created_utc: Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + manifest_count, + package_count: usize::try_from(package_count) + .map_err(|_| PublisherError::Invalid("negative package count in generated index".to_owned()))?, + index_hash, + inventory: inventory.files.clone(), + artifacts, + }) +} + +fn verify_snapshot_path(snapshot: &Path) -> Result<()> { + reject_reparse_components(snapshot)?; + require_directory(snapshot)?; + let snapshot_file = snapshot.join("snapshot.json"); + require_regular_file(&snapshot_file)?; + let metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot_file)?)?; + validate_format_version(metadata.artifact_format_version, "snapshot")?; + if metadata.schema_version != 1 + || metadata.build_id != snapshot.file_name().and_then(|v| v.to_str()).unwrap_or_default() + { + return Err(PublisherError::Invalid( + "snapshot identity or schema version is invalid".to_owned(), + )); + } + validate_build_id(&metadata.build_id)?; + if build_identifier(&Inventory { + files: metadata.inventory.clone(), + manifests: Vec::new(), + })? != metadata.build_id + || metadata.inventory.values().any(|hash| !is_lower_hex_hash(hash)) + { + return Err(PublisherError::Invalid( + "snapshot input inventory does not match its build identifier".to_owned(), + )); + } + let feed = snapshot.join("feed"); + require_directory(&feed)?; + for (relative, expected_hash) in &metadata.artifacts { + let path = safe_join(&feed, relative)?; + let metadata = fs::symlink_metadata(&path)?; + if is_reparse_point(&metadata) || !metadata.is_file() { + return Err(PublisherError::Invalid(format!( + "snapshot contains an unsafe path: {relative}" + ))); + } + let bytes = fs::read(&path)?; + let found = sha256_hex(&bytes); + if &found != expected_hash { + return Err(PublisherError::Invalid(format!( + "snapshot file hash mismatch: {relative}" + ))); + } + } + let found_files = hash_tree(&feed)?; + if found_files != metadata.artifacts { + return Err(PublisherError::Invalid( + "snapshot contains missing or untracked files".to_owned(), + )); + } + if metadata.artifacts.values().any(|hash| !is_lower_hex_hash(hash)) { + return Err(PublisherError::Invalid( + "snapshot artifact inventory contains an invalid hash".to_owned(), + )); + } + let manifest_count = metadata + .artifacts + .keys() + .filter(|path| path.starts_with("manifests/")) + .count(); + if manifest_count != metadata.manifest_count { + return Err(PublisherError::Invalid( + "snapshot manifest count does not match its artifact inventory".to_owned(), + )); + } + let index = snapshot.join("Public").join("index.db"); + let public = index + .parent() + .ok_or_else(|| PublisherError::Invalid("snapshot index path has no parent".to_owned()))?; + require_directory(public)?; + require_regular_file(&index)?; + if !is_lower_hex_hash(&metadata.index_hash) || sha256_hex(&fs::read(&index)?) != metadata.index_hash { + return Err(PublisherError::Invalid( + "snapshot v2 index hash does not match snapshot metadata".to_owned(), + )); + } + let db = Connection::open_with_flags(index, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let integrity: String = db.query_row("PRAGMA integrity_check", [], |row| row.get(0))?; + if integrity != "ok" { + return Err(PublisherError::Invalid(format!( + "SQLite integrity check failed: {integrity}" + ))); + } + let (major, minor): (String, String) = db.query_row( + "SELECT (SELECT value FROM metadata WHERE name='majorVersion'), (SELECT value FROM metadata WHERE name='minorVersion')", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if major != "2" || minor != "0" { + return Err(PublisherError::Invalid(format!( + "snapshot index has unsupported schema version {major}.{minor}" + ))); + } + let package_count: i64 = db.query_row("SELECT COUNT(*) FROM packages", [], |row| row.get(0))?; + if usize::try_from(package_count).ok() != Some(metadata.package_count) { + return Err(PublisherError::Invalid( + "snapshot package count does not match its index".to_owned(), + )); + } + let mut statement = db.prepare("SELECT id,hash FROM packages")?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let id: String = row.get(0)?; + let hash: Vec = row.get(1)?; + if hash.len() != 32 { + return Err(PublisherError::Invalid(format!( + "package {id} has an invalid sidecar hash length" + ))); + } + let hash_hex = hash.iter().map(|byte| format!("{byte:02x}")).collect::(); + let relative = format!("packages/{id}/{}/versionData.mszyml", &hash_hex[..8]); + let sidecar = safe_join(&feed, &relative)?; + let bytes = fs::read(sidecar).map_err(|error| { + PublisherError::Invalid(format!("package {id} version-data sidecar is missing: {error}")) + })?; + if sha256_hex(&bytes) != hash_hex { + return Err(PublisherError::Invalid(format!( + "package {id} version-data hash mismatch" + ))); + } + let version_data = decompress_mszip(&bytes)?; + let value: Value = serde_yaml::from_slice(&version_data)?; + let version_data_map = value + .as_mapping() + .ok_or_else(|| PublisherError::Invalid(format!("package {id} version data is not a mapping")))?; + if required_string(version_data_map, "sV", Path::new(""))? != "1.0" { + return Err(PublisherError::Invalid(format!( + "package {id} has an unsupported version-data schema" + ))); + } + let versions = map_get(version_data_map, "vD") + .and_then(Value::as_sequence) + .ok_or_else(|| PublisherError::Invalid(format!("package {id} version data has no vD entries")))?; + let mut seen_versions = Vec::new(); + for entry in versions { + let map = entry + .as_mapping() + .ok_or_else(|| PublisherError::Invalid(format!("invalid version entry for {id}")))?; + let version = required_string(map, "v", Path::new(""))?; + if seen_versions + .iter() + .any(|previous: &String| compare_winget_versions(previous, &version).is_eq()) + { + return Err(PublisherError::Invalid(format!( + "duplicate version {version} in version data for {id}" + ))); + } + seen_versions.push(version.clone()); + let path = required_string(map, "rP", Path::new(""))?; + let expected = required_string(map, "s256H", Path::new(""))?; + if !is_lower_hex_hash(&expected) { + return Err(PublisherError::Invalid(format!( + "package {id} version data contains an invalid manifest hash" + ))); + } + let manifest = safe_join(&feed, &path)?; + let manifest_bytes = fs::read(manifest)?; + if sha256_hex(&manifest_bytes) != expected { + return Err(PublisherError::Invalid(format!( + "package {id} manifest hash mismatch: {path}" + ))); + } + let value: Value = serde_yaml::from_slice(&manifest_bytes)?; + let manifest_map = as_mapping(&value)?; + if required_string(manifest_map, "PackageIdentifier", Path::new(""))? != id + || required_string(manifest_map, "PackageVersion", Path::new(""))? != version + { + return Err(PublisherError::Invalid(format!( + "version-data manifest identity mismatch for {id} {version}" + ))); + } + } + } + Ok(()) +} + +/// Builds the report for a verified snapshot; `changes` is `(changed_files, removed_files)`. +fn report_from_snapshot(snapshot: &Path, build_id: &str, changes: (usize, usize)) -> Result { + let metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; + Ok(BuildReport { + build_id: build_id.to_owned(), + snapshot_directory: display_path(snapshot), + changed_files: changes.0, + removed_files: changes.1, + packages: metadata.package_count, + manifests: metadata.manifest_count, + files: metadata.artifacts.len() + 1, + }) +} + +fn json_count(value: &serde_json::Value, property: &str) -> Result { + let count = value[property] + .as_u64() + .ok_or_else(|| PublisherError::Invalid(format!("publisher summary is missing {property}")))?; + usize::try_from(count).map_err(|_| PublisherError::Invalid(format!("publisher summary {property} is too large"))) +} + +fn record_build(state: &Path, build_id: &str, snapshot: &SnapshotMetadata) -> Result<()> { + let mut connection = open_state(state, true)?; + let transaction = connection.transaction()?; + let summary = serde_json::json!({ + "manifestCount": snapshot.manifest_count, + "packageCount": snapshot.package_count, + "createdUtc": snapshot.created_utc, + "snapshot": format!("builds/{build_id}"), + }); + transaction.execute( + "INSERT OR REPLACE INTO builds(id,inventory,summary) VALUES(?1,?2,?3)", + params![ + build_id, + canonical_inventory_json(&snapshot.inventory)?, + serde_json::to_string(&summary)? + ], + )?; + transaction.execute( + "INSERT INTO metadata(name,value) VALUES('lastBuildId',?1) ON CONFLICT(name) DO UPDATE SET value=excluded.value", + [build_id], + )?; + transaction.commit()?; + Ok(()) +} + +fn open_state(state: &Path, create: bool) -> Result { + let path = state.join("publisher.db"); + let existed = path.exists(); + match fs::symlink_metadata(&path) { + Ok(_) => require_regular_file(&path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + if !create && !path.is_file() { + return Err(PublisherError::Invalid(format!( + "publisher state does not exist: {}", + path.display() + ))); + } + if create && !existed { + drop(OpenOptions::new().write(true).create_new(true).open(&path)?); + } else if create { + // Validate existing files through a read-only connection before opening a writer. + drop(open_state(state, false)?); + } + let connection = if create { + Connection::open(&path)? + } else { + Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)? + }; + if create && !existed { + connection.execute_batch(STATE_SCHEMA)?; + connection.execute( + "INSERT OR IGNORE INTO metadata(name,value) VALUES('schemaVersion','1')", + [], + )?; + connection.execute( + "INSERT INTO metadata(name,value) VALUES('artifactFormatVersion',?1)", + [PUBLISHER_FORMAT_VERSION.to_string()], + )?; + } + let version: i64 = connection.query_row("PRAGMA user_version", [], |row| row.get(0))?; + if version != 1 { + return Err(PublisherError::Invalid(format!( + "unsupported publisher state schema version: {version}" + ))); + } + let schema_version: Option = connection + .query_row("SELECT value FROM metadata WHERE name='schemaVersion'", [], |row| { + row.get(0) + }) + .optional()?; + if schema_version.as_deref() != Some("1") { + return Err(PublisherError::Invalid(format!( + "unsupported or missing publisher state schemaVersion: {}", + schema_version.as_deref().unwrap_or("") + ))); + } + let format_version: Option = connection + .query_row( + "SELECT value FROM metadata WHERE name='artifactFormatVersion'", + [], + |row| row.get(0), + ) + .optional()?; + let format_version = format_version + .as_deref() + .unwrap_or("1") + .parse::() + .map_err(|_| PublisherError::Invalid("publisher state artifactFormatVersion is invalid".to_owned()))?; + validate_format_version(format_version, "publisher state")?; + let tables: Vec = { + let mut statement = connection + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name")?; + statement + .query_map([], |row| row.get(0))? + .collect::>()? + }; + if tables != ["builds", "metadata"] { + return Err(PublisherError::Invalid( + "publisher state has an unexpected schema".to_owned(), + )); + } + connection.prepare("SELECT name,value FROM metadata LIMIT 0")?; + connection.prepare("SELECT id,inventory,summary FROM builds LIMIT 0")?; + Ok(connection) +} + +fn validate_recorded_snapshot(connection: &Connection, snapshot: &SnapshotMetadata) -> Result<()> { + validate_format_version(snapshot.artifact_format_version, "snapshot")?; + let record: Option<(String, String)> = connection + .query_row( + "SELECT inventory,summary FROM builds WHERE id=?1", + [&snapshot.build_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + let (inventory, summary) = record.ok_or_else(|| { + PublisherError::Invalid(format!( + "snapshot {} is not recorded in publisher state", + snapshot.build_id + )) + })?; + let inventory: BTreeMap = serde_json::from_str(&inventory)?; + let summary: serde_json::Value = serde_json::from_str(&summary)?; + if inventory != snapshot.inventory + || json_count(&summary, "manifestCount")? != snapshot.manifest_count + || json_count(&summary, "packageCount")? != snapshot.package_count + || summary["createdUtc"].as_str() != Some(snapshot.created_utc.as_str()) + { + return Err(PublisherError::Invalid( + "snapshot metadata does not match its recorded build".to_owned(), + )); + } + Ok(()) +} + +fn validate_format_version(version: u32, subject: &str) -> Result<()> { + if version != PUBLISHER_FORMAT_VERSION { + return Err(PublisherError::Invalid(format!( + "{subject} uses unsupported artifact format version {}; use a new --state directory; existing state and snapshots are preserved", + if version == 0 { 1 } else { version } + ))); + } + Ok(()) +} + +fn remove_owned_staging(path: &Path) -> Result<()> { + reject_reparse_components(path)?; + require_directory(path)?; + let mut files = Vec::new(); + gather_files(path, path, &mut files)?; + fs::remove_dir_all(path)?; + Ok(()) +} + +fn acquire_destination_lock(destination: &Path) -> Result { + let parent = destination + .parent() + .ok_or_else(|| PublisherError::Invalid("destination must not be a volume root".to_owned()))?; + let name = destination + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| PublisherError::Invalid("destination name must be Unicode".to_owned()))?; + reject_reparse_components(parent)?; + fs::create_dir_all(parent)?; + let lock = parent.join(format!("{name}.pinget-publisher.lock")); + let result = (|| { + reject_reparse_components(&lock)?; + if lock.exists() { + require_regular_file(&lock)?; + } + Ok(WriterLock { + _file: lock_file(&lock)?, + }) + })(); + result.map_err(|error: PublisherError| { + PublisherError::Invalid(format!( + "destination writer lock unavailable at {}: {error}", + lock.display() + )) + }) +} + +fn is_published(connection: &Connection, build_id: &str) -> Result { + let published: Option = connection + .query_row( + "SELECT value FROM metadata WHERE name='lastPublishedBuildId'", + [], + |row| row.get(0), + ) + .optional()?; + Ok(published.as_deref() == Some(build_id)) +} + +fn acquire_lock(state: &Path) -> Result { + let lock_path = state.join(".publisher.lock"); + match fs::symlink_metadata(&lock_path) { + Ok(_) => require_regular_file(&lock_path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let file = lock_file(&lock_path)?; + Ok(WriterLock { _file: file }) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OwnerMarker { + schema_version: u32, + owner: String, + package_name: String, + publisher: String, + package_version: String, + build_id: String, + index_hash: String, + unsigned_development: bool, + published_utc: String, + artifacts: BTreeMap, +} + +#[derive(Default)] +struct PublicationRollback { + files: Vec, + directories: Vec, + committed: bool, +} + +impl PublicationRollback { + fn create_directories(&mut self, root: &Path, directory: &Path) -> Result<()> { + let relative = directory + .strip_prefix(root) + .map_err(|_| PublisherError::Invalid("publication directory escaped root".to_owned()))?; + let mut current = root.to_path_buf(); + for component in relative.components() { + current.push(component); + reject_reparse_components(¤t)?; + if !current.exists() { + fs::create_dir(¤t)?; + self.directories.push(current.clone()); + } + } + Ok(()) + } +} + +impl Drop for PublicationRollback { + fn drop(&mut self) { + if !self.committed { + for file in self.files.iter().rev() { + let _ = fs::remove_file(file); + } + for directory in self.directories.iter().rev() { + let _ = fs::remove_dir(directory); + } + } + } +} + +fn package_and_publish( + snapshot: &Path, + temporary: &Path, + destination: &Path, + options: &PublishOptions, + build_id: &str, + unsigned_development: bool, + prior: Option, +) -> Result { + let metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; + let package_root = temporary.join("package"); + fs::create_dir_all(package_root.join("Public"))?; + fs::copy( + snapshot.join("Public").join("index.db"), + package_root.join("Public").join("index.db"), + )?; + create_appx_assets(&package_root, &options.name, &options.publisher, &options.version)?; + let msix = temporary.join("source2.msix"); + let makeappx = find_tool("makeappx.exe", options.makeappx.as_deref()).ok_or_else(|| { + PublisherError::Tool("makeappx.exe was not found; pass --makeappx or add it to PATH".to_owned()) + })?; + run_tool( + &makeappx, + [ + "pack", + "/d", + &package_root.to_string_lossy(), + "/p", + &msix.to_string_lossy(), + "/o", + ], + )?; + match &options.signing { + Signing::Pfx { path, password_env } => { + if !path.is_file() { + return Err(PublisherError::Invalid(format!( + "PFX does not exist: {}", + path.display() + ))); + } + if std::env::var_os(password_env).is_none() { + return Err(PublisherError::Invalid(format!( + "password environment variable {password_env} is not set" + ))); + } + validate_certificate_publisher(path, password_env, &options.publisher)?; + let password = std::env::var(password_env).map_err(|_| { + PublisherError::Invalid(format!("password environment variable {password_env} is not set")) + })?; + let signtool = find_tool("signtool.exe", options.signtool.as_deref()).ok_or_else(|| { + PublisherError::Tool("signtool.exe was not found; pass --signtool or add it to PATH".to_owned()) + })?; + let pfx_path = path.to_string_lossy(); + let msix_path = msix.to_string_lossy(); + let mut arguments = vec!["sign", "/fd", "SHA256", "/f", &pfx_path, "/p", &password]; + if let Some(url) = &options.timestamp_url { + arguments.extend(["/tr", url, "/td", "SHA256"]); + } + arguments.push(&msix_path); + run_sign_tool(&signtool, &arguments, Some(&password))?; + run_tool(&signtool, ["verify", "/pa", "/v", &msix.to_string_lossy()])?; + } + Signing::UnsignedDevelopment => {} + } + let mut marker = prior.unwrap_or_else(|| OwnerMarker { + schema_version: 1, + owner: "Pinget.Publisher".to_owned(), + package_name: options.name.clone(), + publisher: options.publisher.clone(), + package_version: options.version.clone(), + build_id: build_id.to_owned(), + index_hash: metadata.index_hash.clone(), + unsigned_development, + published_utc: String::new(), + artifacts: BTreeMap::new(), + }); + marker.package_version = options.version.clone(); + marker.build_id = build_id.to_owned(); + marker.index_hash = metadata.index_hash.clone(); + marker.unsigned_development = unsigned_development; + verify_msix_contents(&msix, &marker)?; + let package_hash = sha256_hex(&fs::read(&msix)?); + let mut rollback = PublicationRollback::default(); + for relative in metadata.artifacts.keys() { + let target = safe_join(destination, relative)?; + if target.exists() { + let hash = sha256_hex(&fs::read(&target)?); + if metadata.artifacts.get(relative) != Some(&hash) { + return Err(PublisherError::Invalid(format!( + "immutable publication path conflicts: {relative}" + ))); + } + } + } + for relative in metadata.artifacts.keys() { + let target = safe_join(destination, relative)?; + if !target.exists() { + let source = safe_join(&snapshot.join("feed"), relative)?; + let parent = target + .parent() + .ok_or_else(|| PublisherError::Invalid(format!("invalid publication path {relative}")))?; + rollback.create_directories(destination, parent)?; + let mut output = OpenOptions::new().write(true).create_new(true).open(&target)?; + rollback.files.push(target.clone()); + let mut input = File::open(source)?; + std::io::copy(&mut input, &mut output)?; + output.sync_all()?; + drop(output); + if metadata.artifacts.get(relative) != Some(&sha256_hex(&fs::read(&target)?)) { + return Err(PublisherError::Invalid(format!( + "copied publication hash mismatch: {relative}" + ))); + } + } + } + let target_msix = destination.join("source2.msix"); + let staged_package = stage_package(&msix, destination, build_id)?; + rollback.files.push(staged_package.clone()); + + marker.owner = "Pinget.Publisher".to_owned(); + marker.package_name = options.name.clone(); + marker.publisher = options.publisher.clone(); + marker.package_version = options.version.clone(); + marker.build_id = build_id.to_owned(); + marker.index_hash = metadata.index_hash; + marker.unsigned_development = unsigned_development; + marker.published_utc = Utc::now().to_rfc3339(); + marker.artifacts = published_inventory_excluding(destination, &[temporary, staged_package.as_path()])?; + marker.artifacts.insert("source2.msix".to_owned(), package_hash); + let old_marker_path = destination.join(".pinget-publisher.json"); + let marker_backup = temporary.join("prior-marker.json"); + let has_old_marker = old_marker_path.exists(); + if has_old_marker { + fs::copy(&old_marker_path, &marker_backup)?; + OpenOptions::new().write(true).open(&marker_backup)?.sync_all()?; + } + let marker_temp = destination.join(".pinget-publisher.json.new"); + write_json(&marker_temp, &marker)?; + rollback.files.push(marker_temp.clone()); + if let Err(error) = atomic_replace(&marker_temp, &destination.join(".pinget-publisher.json")) + .and_then(|()| atomic_replace(&staged_package, &target_msix)) + { + if has_old_marker { + let _ = atomic_replace(&marker_backup, &old_marker_path); + } else { + let _ = fs::remove_file(destination.join(".pinget-publisher.json")); + } + let _ = fs::remove_file(marker_temp); + let _ = fs::remove_file(staged_package); + return Err(error); + } + rollback.committed = true; + + // The marker is the publication commit record. Persist a matching state row. + let state = options.state.canonicalize()?; + mark_published(&state, build_id)?; + fs::remove_dir_all(temporary)?; + let verified = verify_destination_with_tool(destination, options.signtool.as_deref())?; + Ok(PublishReport { + valid: true, + subject: verified.subject, + artifact_count: verified.artifact_count, + build_id: build_id.to_owned(), + destination: display_path(destination), + unsigned_development, + reused: false, + }) +} + +fn stage_package(source: &Path, destination: &Path, build_id: &str) -> Result { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| PublisherError::Invalid(format!("system clock is before Unix epoch: {error}")))? + .as_nanos(); + let staged = destination.join(format!(".source2.{build_id}.{}.{}.new", std::process::id(), timestamp)); + let mut output = OpenOptions::new().write(true).create_new(true).open(&staged)?; + let result = (|| -> Result<()> { + let mut input = File::open(source)?; + std::io::copy(&mut input, &mut output)?; + output.sync_all()?; + Ok(()) + })(); + if let Err(error) = result { + drop(output); + let _ = fs::remove_file(&staged); + return Err(error); + } + Ok(staged) +} + +/// Removes the verbatim \\?\ prefix added by canonicalization when the plain form is equivalent. +fn display_path(path: &Path) -> PathBuf { + let text = path.to_string_lossy(); + if let Some(rest) = text.strip_prefix(r"\\?\UNC\") { + return PathBuf::from(format!(r"\\{rest}")); + } + if let Some(rest) = text.strip_prefix(r"\\?\") + && rest.as_bytes().get(1) == Some(&b':') + { + return PathBuf::from(rest); + } + path.to_path_buf() +} + +fn mark_published(state: &Path, build_id: &str) -> Result<()> { + let mut connection = open_state(state, true)?; + let transaction = connection.transaction()?; + transaction.execute( + "INSERT INTO metadata(name,value) VALUES('lastPublishedBuildId',?1) ON CONFLICT(name) DO UPDATE SET value=excluded.value", + [build_id], + )?; + transaction.commit()?; + Ok(()) +} + +#[cfg(windows)] +fn atomic_replace(source: &Path, destination: &Path) -> Result<()> { + use std::os::windows::ffi::OsStrExt; + const MOVEFILE_REPLACE_EXISTING: u32 = 0x1; + const MOVEFILE_WRITE_THROUGH: u32 = 0x8; + #[link(name = "kernel32")] + unsafe extern "system" { + fn MoveFileExW(existing: *const u16, new: *const u16, flags: u32) -> i32; + } + let source_wide: Vec = source.as_os_str().encode_wide().chain(Some(0)).collect(); + let destination_wide: Vec = destination.as_os_str().encode_wide().chain(Some(0)).collect(); + // SAFETY: both paths are NUL-terminated UTF-16 buffers that remain live for the call. + if unsafe { + MoveFileExW( + source_wide.as_ptr(), + destination_wide.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + } == 0 + { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) +} + +#[cfg(not(windows))] +fn atomic_replace(source: &Path, destination: &Path) -> Result<()> { + fs::rename(source, destination)?; + Ok(()) +} + +#[cfg(windows)] +fn lock_file(path: &Path) -> Result { + use std::os::windows::fs::OpenOptionsExt; + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .share_mode(0) + .open(path) + .map_err(|error| { + if error.kind() == std::io::ErrorKind::PermissionDenied { + PublisherError::Invalid(format!("another publisher is active: {}", path.display())) + } else { + error.into() + } + }) +} + +#[cfg(not(windows))] +fn lock_file(path: &Path) -> Result { + use fs2::FileExt; + let file = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path)?; + file.try_lock_exclusive() + .map_err(|_| PublisherError::Invalid(format!("another publisher is active: {}", path.display())))?; + Ok(file) +} + +fn verify_msix_contents(package_path: &Path, marker: &OwnerMarker) -> Result<()> { + let file = File::open(package_path)?; + let mut archive = ZipArchive::new(file) + .map_err(|error| PublisherError::Invalid(format!("could not read source MSIX: {error}")))?; + let index_hash = { + let mut index = archive.by_name("Public/index.db").map_err(|error| { + PublisherError::Invalid(format!("source MSIX does not contain Public/index.db: {error}")) + })?; + let mut bytes = Vec::new(); + index.read_to_end(&mut bytes)?; + sha256_hex(&bytes) + }; + if index_hash != marker.index_hash { + return Err(PublisherError::Invalid( + "source MSIX contains a v2 index different from the published build".to_owned(), + )); + } + let manifest = { + let mut entry = archive.by_name("AppxManifest.xml").map_err(|error| { + PublisherError::Invalid(format!("source MSIX does not contain AppxManifest.xml: {error}")) + })?; + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + bytes + }; + let mut reader = Reader::from_reader(manifest.as_slice()); + reader.config_mut().trim_text(true); + let mut identity = None; + let mut extension = None; + loop { + match reader.read_event() { + Ok(Event::Start(element) | Event::Empty(element)) => { + let local_name = element.local_name(); + if local_name.as_ref() == b"Identity" { + if identity.replace(xml_attributes(&reader, &element)?).is_some() { + return Err(PublisherError::Invalid( + "source MSIX has multiple package identities".to_owned(), + )); + } + } else if local_name.as_ref() == b"AppExtension" { + let attributes = xml_attributes(&reader, &element)?; + match (attributes.get("Name").map(String::as_str), extension.is_some()) { + (Some("com.microsoft.winget.source"), true) => { + return Err(PublisherError::Invalid( + "source MSIX has multiple WinGet source extensions".to_owned(), + )); + } + (Some("com.microsoft.winget.source"), false) => { + extension = Some(attributes); + } + _ => {} + } + } + } + Ok(Event::Eof) => break, + Ok(_) => {} + Err(error) => { + return Err(PublisherError::Invalid(format!( + "could not parse source MSIX manifest: {error}" + ))); + } + } + } + let identity = + identity.ok_or_else(|| PublisherError::Invalid("source MSIX manifest has no Identity element".to_owned()))?; + if identity.get("Name") != Some(&marker.package_name) + || identity.get("Publisher") != Some(&marker.publisher) + || identity.get("Version") != Some(&marker.package_version) + { + return Err(PublisherError::Invalid( + "source MSIX identity does not match the publisher ownership marker".to_owned(), + )); + } + let extension = extension + .ok_or_else(|| PublisherError::Invalid("source MSIX has no WinGet preindexed source extension".to_owned()))?; + if extension.get("PublicFolder").is_none_or(|value| value != "Public") { + return Err(PublisherError::Invalid( + "source MSIX extension does not expose the Public folder".to_owned(), + )); + } + Ok(()) +} + +fn xml_attributes( + reader: &Reader, + element: &quick_xml::events::BytesStart<'_>, +) -> Result> { + let mut attributes = BTreeMap::new(); + for attribute in element.attributes() { + let attribute = + attribute.map_err(|error| PublisherError::Invalid(format!("invalid AppxManifest attribute: {error}")))?; + let name = String::from_utf8_lossy(attribute.key.local_name().as_ref()).into_owned(); + let value = attribute + .decode_and_unescape_value(reader.decoder()) + .map_err(|error| PublisherError::Invalid(format!("invalid AppxManifest attribute: {error}")))? + .into_owned(); + attributes.insert(name, value); + } + Ok(attributes) +} + +fn verify_destination(destination: &Path) -> Result { + verify_destination_with_tool(destination, None) +} + +fn verify_destination_with_tool(destination: &Path, signtool_path: Option<&Path>) -> Result { + reject_reparse_components(destination)?; + let destination = fs::canonicalize(destination)?; + let marker = read_owner_marker(&destination.join(".pinget-publisher.json"))? + .ok_or_else(|| PublisherError::Invalid("destination has no Pinget ownership marker".to_owned()))?; + for (relative, expected) in &marker.artifacts { + let path = safe_join(&destination, relative)?; + let bytes = fs::read(&path)?; + let found = sha256_hex(&bytes); + if &found != expected { + return Err(PublisherError::Invalid(format!( + "published file hash mismatch: {relative}" + ))); + } + } + let current = published_inventory(&destination)?; + if current != marker.artifacts { + return Err(PublisherError::Invalid( + "published directory has missing or untracked files".to_owned(), + )); + } + if !is_lower_hex_hash(&marker.index_hash) { + return Err(PublisherError::Invalid( + "publisher marker contains an invalid index hash".to_owned(), + )); + } + verify_msix_contents(&destination.join("source2.msix"), &marker)?; + if !marker.unsigned_development { + if !cfg!(windows) { + return Err(PublisherError::Tool( + "MSIX signature verification requires Windows signtool on this platform".to_owned(), + )); + } + let signtool = find_tool("signtool.exe", signtool_path) + .ok_or_else(|| PublisherError::Tool("signtool.exe was not found on PATH".to_owned()))?; + let package_path = destination.join("source2.msix"); + run_tool(&signtool, ["verify", "/pa", "/v", &package_path.to_string_lossy()])?; + } + Ok(VerifyReport { + valid: true, + subject: "published WinGet v2 static source".to_owned(), + artifact_count: marker.artifacts.len(), + build_id: Some(marker.build_id), + }) +} + +fn ensure_owned_destination(destination: &Path) -> Result<()> { + let marker = destination.join(".pinget-publisher.json"); + let entries: Vec<_> = fs::read_dir(destination)?.collect::>>()?; + if entries.is_empty() || marker.is_file() { + return Ok(()); + } + Err(PublisherError::Invalid(format!( + "refusing to publish into non-empty unowned destination: {}", + destination.display() + ))) +} + +fn read_owner_marker(path: &Path) -> Result> { + match fs::symlink_metadata(path) { + Ok(_) => require_regular_file(path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + } + let bytes = fs::read(path)?; + if bytes.len() > 8 * 1024 * 1024 { + return Err(PublisherError::Invalid( + "publication ownership marker exceeds the 8 MiB limit".to_owned(), + )); + } + let marker: OwnerMarker = serde_json::from_slice(&bytes)?; + if marker.schema_version != 1 { + return Err(PublisherError::Invalid(format!( + "unsupported publication inventory schema version {}", + marker.schema_version + ))); + } + if marker.owner != "Pinget.Publisher" + || marker.package_name.is_empty() + || marker.publisher.is_empty() + || !is_lower_hex_hash(&marker.index_hash) + { + return Err(PublisherError::Invalid( + "publisher ownership marker identity is invalid".to_owned(), + )); + } + validate_build_id(&marker.build_id)?; + validate_msix_version(&marker.package_version)?; + for (relative, hash) in &marker.artifacts { + validate_relative_path(Path::new(relative))?; + if relative == ".pinget-publisher.json" || !is_lower_hex_hash(hash) { + return Err(PublisherError::Invalid( + "publication ownership marker contains an invalid inventory entry".to_owned(), + )); + } + } + if !marker.artifacts.contains_key("source2.msix") { + return Err(PublisherError::Invalid( + "publisher ownership marker does not list source2.msix".to_owned(), + )); + } + Ok(Some(marker)) +} + +fn published_inventory(root: &Path) -> Result> { + published_inventory_excluding(root, &[]) +} + +fn published_inventory_excluding(root: &Path, excluded: &[&Path]) -> Result> { + let mut files = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + if excluded.iter().any(|path| *path == entry.path()) { + continue; + } + if entry.file_name() == ".pinget-publisher.json" || entry.file_name() == ".pinget-publisher.json.new" { + continue; + } + gather_files(root, &entry.path(), &mut files)?; + } + files.sort(); + let mut inventory = BTreeMap::new(); + for path in files { + let relative = path + .strip_prefix(root) + .map_err(|_| PublisherError::Invalid("published path escaped destination".to_owned()))?; + let key = relative.to_string_lossy().replace('\\', "/"); + inventory.insert(key, sha256_hex(&fs::read(path)?)); + } + Ok(inventory) +} + +fn create_appx_assets(root: &Path, name: &str, publisher: &str, version: &str) -> Result<()> { + fs::create_dir_all(root.join("Assets"))?; + let escaped_publisher = escape_xml(publisher); + let manifest = format!( + "\nPinget preindexed source{escaped_publisher}Assets/StoreLogo.png\n" + ); + fs::write(root.join("AppxManifest.xml"), manifest)?; + write_png_logo(&root.join("Assets").join("StoreLogo.png"), 50)?; + write_png_logo(&root.join("Assets").join("Logo150.png"), 150)?; + write_png_logo(&root.join("Assets").join("Logo44.png"), 44)?; + Ok(()) +} + +fn escape_xml(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +fn write_png_logo(path: &Path, size: usize) -> Result<()> { + use flate2::Compression; + use flate2::write::ZlibEncoder; + + let mut row = Vec::with_capacity(size * 4 + 1); + row.push(0); + for _ in 0..size { + row.extend_from_slice(&[0x00, 0x78, 0xd7, 0xff]); + } + let mut scanlines = Vec::with_capacity(size * (size * 4 + 1)); + for _ in 0..size { + scanlines.extend_from_slice(&row); + } + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&scanlines)?; + let compressed = encoder.finish()?; + + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + let mut header = Vec::with_capacity(13); + let size = u32::try_from(size) + .map_err(|_| PublisherError::Invalid("generated PNG dimensions are too large".to_owned()))?; + header.extend_from_slice(&size.to_be_bytes()); + header.extend_from_slice(&size.to_be_bytes()); + header.extend_from_slice(&[8, 6, 0, 0, 0]); + append_png_chunk(&mut png, b"IHDR", &header)?; + append_png_chunk(&mut png, b"IDAT", &compressed)?; + append_png_chunk(&mut png, b"IEND", &[])?; + fs::write(path, png)?; + Ok(()) +} + +fn append_png_chunk(output: &mut Vec, kind: &[u8; 4], data: &[u8]) -> Result<()> { + let length = u32::try_from(data.len()) + .map_err(|_| PublisherError::Invalid("generated PNG chunk is too large".to_owned()))?; + output.extend_from_slice(&length.to_be_bytes()); + output.extend_from_slice(kind); + output.extend_from_slice(data); + let mut checksum = kind.to_vec(); + checksum.extend_from_slice(data); + output.extend_from_slice(&crc32(&checksum).to_be_bytes()); + Ok(()) +} + +fn crc32(bytes: &[u8]) -> u32 { + let mut crc = !0u32; + for byte in bytes { + crc ^= u32::from(*byte); + for _ in 0..8 { + crc = (crc >> 1) ^ (0xedb8_8320 & (0u32.wrapping_sub(crc & 1))); + } + } + !crc +} + +fn run_tool(tool: &Path, args: [&str; N]) -> Result<()> { + let output = Command::new(tool).args(args).output()?; + if output.status.success() { + return Ok(()); + } + Err(PublisherError::Tool(format!( + "{} failed ({}): {}", + tool.display(), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))) +} + +fn run_sign_tool(tool: &Path, args: &[&str], secret: Option<&str>) -> Result<()> { + let output = Command::new(tool).args(args).output()?; + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr); + let stderr = secret.map_or_else(|| stderr.to_string(), |secret| stderr.replace(secret, "")); + Err(PublisherError::Tool(format!( + "{} failed ({}): {}", + tool.display(), + output.status, + stderr.trim() + ))) +} + +fn validate_certificate_publisher(path: &Path, password_env: &str, publisher: &str) -> Result<()> { + let powershell = find_tool("powershell.exe", None).ok_or_else(|| { + PublisherError::Tool("Windows PowerShell is required to inspect the signing certificate".to_owned()) + })?; + let script = "$ErrorActionPreference='Stop'; try { $p=[Environment]::GetEnvironmentVariable($env:PINGET_PUBLISHER_PASSWORD_NAME); $c=[System.Security.Cryptography.X509Certificates.X509Certificate2]::new($env:PINGET_PUBLISHER_PFX_PATH,$p,[System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet); $now=[DateTime]::UtcNow; $eku=$c.Extensions | Where-Object { $_ -is [System.Security.Cryptography.X509Certificates.X509EnhancedKeyUsageExtension] }; $codeSigning=$false; if ($eku) { $codeSigning=@($eku.EnhancedKeyUsages | Where-Object { $_.Value -eq '1.3.6.1.5.5.7.3.3' }).Count -gt 0 }; $result=@{ subject=$c.Subject; hasPrivateKey=$c.HasPrivateKey; validNow=($now -ge $c.NotBefore.ToUniversalTime() -and $now -le $c.NotAfter.ToUniversalTime()); codeSigning=$codeSigning }; [Console]::Out.Write((ConvertTo-Json -InputObject $result -Compress)) } catch { [Console]::Error.WriteLine('Unable to inspect the signing certificate.'); exit 1 }"; + let output = Command::new(powershell) + .args(["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", script]) + .env("PINGET_PUBLISHER_PFX_PATH", path) + .env("PINGET_PUBLISHER_PASSWORD_NAME", password_env) + .output()?; + if !output.status.success() { + return Err(PublisherError::Invalid( + "unable to inspect the PFX signing certificate; confirm its password and certificate contents".to_owned(), + )); + } + let certificate: serde_json::Value = serde_json::from_slice(&output.stdout) + .map_err(|_| PublisherError::Invalid("unable to parse signing certificate details".to_owned()))?; + let subject = certificate["subject"].as_str().unwrap_or_default(); + if !subject.eq_ignore_ascii_case(publisher) { + return Err(PublisherError::Invalid(format!( + "PFX certificate subject does not match --publisher (expected {publisher:?}, found {subject:?})" + ))); + } + if certificate["hasPrivateKey"] != true { + return Err(PublisherError::Invalid( + "PFX certificate does not contain a private key".to_owned(), + )); + } + if certificate["validNow"] != true { + return Err(PublisherError::Invalid( + "PFX certificate is not currently valid".to_owned(), + )); + } + if certificate["codeSigning"] != true { + return Err(PublisherError::Invalid( + "PFX certificate must include the Code Signing enhanced key usage".to_owned(), + )); + } + Ok(()) +} + +fn find_tool(name: &str, explicit: Option<&Path>) -> Option { + if let Some(path) = explicit { + return path.is_file().then(|| path.to_path_buf()); + } + if let Some(path_env) = std::env::var_os("PATH") + && let Some(path) = std::env::split_paths(&path_env) + .map(|directory| directory.join(name)) + .find(|path| path.is_file()) + { + return Some(path); + } + if !cfg!(windows) || !["makeappx.exe", "signtool.exe"].contains(&name) { + return None; + } + let root = PathBuf::from(std::env::var_os("ProgramFiles(x86)")?) + .join("Windows Kits") + .join("10") + .join("bin"); + let host_arch = std::env::var("PROCESSOR_ARCHITEW6432") + .or_else(|_| std::env::var("PROCESSOR_ARCHITECTURE")) + .unwrap_or_default(); + let architectures: &[&str] = if host_arch.eq_ignore_ascii_case("ARM64") || cfg!(target_arch = "aarch64") { + &["arm64", "x64", "x86"] + } else { + &["x64", "x86"] + }; + find_sdk_tool(&root, name, architectures) +} + +fn find_sdk_tool(root: &Path, name: &str, architectures: &[&str]) -> Option { + let mut versions: Vec<_> = fs::read_dir(root) + .ok()? + .filter_map(std::result::Result::ok) + .filter_map(|entry| { + let version: Vec = entry + .file_name() + .to_str()? + .split('.') + .map(str::parse) + .collect::>() + .ok()?; + (version.len() == 4).then_some((version, entry.path())) + }) + .collect(); + versions.sort_by(|left, right| right.0.cmp(&left.0)); + versions + .into_iter() + .flat_map(|(_, directory)| architectures.iter().map(move |arch| directory.join(arch).join(name))) + .find(|path| path.is_file()) +} + +fn write_json(path: &Path, value: &impl Serialize) -> Result<()> { + let bytes = serde_json::to_vec_pretty(value)?; + let mut file = OpenOptions::new().write(true).create_new(true).open(path)?; + let result = (|| { + file.write_all(&bytes)?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) + })(); + if result.is_err() { + drop(file); + let _ = fs::remove_file(path); + } + result +} + +fn write_relative(root: &Path, relative: &str, bytes: &[u8]) -> Result<()> { + let path = safe_join(root, relative)?; + if path.exists() { + let old = fs::read(&path)?; + if old != bytes { + return Err(PublisherError::Invalid(format!( + "content-addressed path already contains different bytes: {relative}" + ))); + } + return Ok(()); + } + let parent = path + .parent() + .ok_or_else(|| PublisherError::Invalid(format!("invalid output path {relative}")))?; + fs::create_dir_all(parent)?; + let mut file = OpenOptions::new().write(true).create_new(true).open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + Ok(()) +} + +fn safe_join(root: &Path, relative: &str) -> Result { + let path = Path::new(relative); + validate_relative_path(path)?; + let target = root.join(path); + if target.starts_with(root) { + reject_reparse_components(&target)?; + Ok(target) + } else { + Err(PublisherError::Invalid(format!("path escapes root: {relative}"))) + } +} + +fn validate_relative_path(path: &Path) -> Result<()> { + if path.as_os_str().is_empty() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(PublisherError::Invalid(format!( + "unsafe relative path: {}", + path.display() + ))); + } + Ok(()) +} + +fn safe_component(value: &str) -> Result { + if value.is_empty() + || value == "." + || value == ".." + || value + .chars() + .any(|character| character == '/' || character == '\\' || character == ':' || character.is_control()) + { + return Err(PublisherError::Invalid(format!("unsafe path component: {value:?}"))); + } + Ok(value.to_owned()) +} + +fn validate_package_identifier(id: &str) -> Result<()> { + safe_component(id)?; + let mut chars = id.chars(); + if id.len() > 128 + || !chars.next().is_some_and(|ch| ch.is_ascii_alphanumeric()) + || !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-')) + || id.trim() != id + { + return Err(PublisherError::Invalid(format!("PackageIdentifier is invalid: {id:?}"))); + } + Ok(()) +} + +fn validate_package_version(version: &str) -> Result<()> { + let mut chars = version.chars(); + if version.len() > 128 + || !chars.next().is_some_and(|ch| ch.is_ascii_alphanumeric()) + || !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '+' | '-')) + { + return Err(PublisherError::Invalid(format!( + "PackageVersion is invalid: {version:?}" + ))); + } + safe_component(version)?; + Ok(()) +} + +fn validate_locale(locale: &str) -> Result<()> { + let mut components = locale.split('-'); + let language = components.next().unwrap_or_default(); + let valid_language = (2..=3).contains(&language.len()) && language.bytes().all(|byte| byte.is_ascii_alphabetic()); + let valid_subtags = + components.all(|part| (2..=8).contains(&part.len()) && part.bytes().all(|byte| byte.is_ascii_alphanumeric())); + if !valid_language || !valid_subtags || locale.len() > 35 { + return Err(PublisherError::Invalid(format!("PackageLocale is invalid: {locale:?}"))); + } + safe_component(locale)?; + Ok(()) +} + +fn validate_yaml_value(value: &Value, depth: usize) -> Result<()> { + if depth > 100 { + return Err(PublisherError::Invalid( + "YAML nesting depth exceeds the 100-level limit".to_owned(), + )); + } + match value { + Value::Mapping(mapping) => { + for (key, child) in mapping { + if key.as_str().is_none_or(str::is_empty) { + return Err(PublisherError::Invalid( + "manifest mapping keys must be non-empty strings".to_owned(), + )); + } + validate_yaml_value(child, depth + 1)?; + } + } + Value::Sequence(sequence) => { + for child in sequence { + validate_yaml_value(child, depth + 1)?; + } + } + _ => {} + } + Ok(()) +} + +fn validate_package_identity(name: &str, publisher: &str) -> Result<()> { + let segments: Vec<_> = name.split('.').collect(); + if name.len() > 50 + || segments.is_empty() + || segments + .iter() + .any(|segment| segment.is_empty() || !segment.bytes().all(|byte| byte.is_ascii_alphanumeric())) + { + return Err(PublisherError::Invalid("invalid MSIX package identity name".to_owned())); + } + if publisher.is_empty() || publisher.contains(['<', '>']) || publisher.chars().any(char::is_control) { + return Err(PublisherError::Invalid("invalid MSIX package publisher".to_owned())); + } + Ok(()) +} + +fn validate_msix_version(version: &str) -> Result<()> { + let parts: Vec<_> = version.split('.').collect(); + if parts.len() != 4 + || parts.iter().any(|part| { + part.is_empty() || !part.bytes().all(|byte| byte.is_ascii_digit()) || part.parse::().is_err() + }) + { + return Err(PublisherError::Invalid(format!( + "MSIX package version must contain four numeric components: {version}" + ))); + } + Ok(()) +} + +fn compare_winget_versions(left: &str, right: &str) -> std::cmp::Ordering { + let left = ParsedVersion::parse(left); + let right = ParsedVersion::parse(right); + if left.latest != right.latest { + return left.latest.cmp(&right.latest); + } + if left.unknown != right.unknown { + return right.unknown.cmp(&left.unknown); + } + for index in 0..left.parts.len().max(right.parts.len()) { + let part = compare_version_parts( + left.parts.get(index).unwrap_or(&VersionPart::ZERO), + right.parts.get(index).unwrap_or(&VersionPart::ZERO), + ); + if part != std::cmp::Ordering::Equal { + return part; + } + } + left.approximation.cmp(&right.approximation) +} + +#[derive(Debug)] +struct ParsedVersion { + latest: bool, + unknown: bool, + approximation: i32, + parts: Vec, +} + +impl ParsedVersion { + fn parse(source: &str) -> Self { + let mut value = source.trim(); + let mut approximation = 0; + if let Some(rest) = value.strip_prefix("< ") { + approximation = -1; + value = rest; + } else if let Some(rest) = value.strip_prefix("> ") { + approximation = 1; + value = rest; + } + let first_digit = value.find(|character: char| character.is_ascii_digit()); + let first_separator = value.find('.'); + if let Some(first_digit) = first_digit + && first_separator.is_none_or(|first_separator| first_digit < first_separator) + { + value = &value[first_digit..]; + } + let mut parts: Vec<_> = value.split('.').map(VersionPart::parse).collect(); + while parts.last().is_some_and(|part| *part == VersionPart::ZERO) { + parts.pop(); + } + let latest = parts.len() == 1 && parts[0].integer == 0 && parts[0].other.eq_ignore_ascii_case("latest"); + let unknown = parts.len() == 1 && parts[0].integer == 0 && parts[0].other.eq_ignore_ascii_case("unknown"); + Self { + latest, + unknown, + approximation, + parts, + } + } +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct VersionPart { + integer: u64, + other: String, +} + +impl VersionPart { + const ZERO: Self = Self { + integer: 0, + other: String::new(), + }; + + fn parse(raw: &str) -> Self { + let part = raw.trim(); + let digits = part.bytes().take_while(u8::is_ascii_digit).count(); + let Some(integer) = part.get(..digits).and_then(|value| value.parse::().ok()) else { + return Self { + integer: 0, + other: part.to_lowercase(), + }; + }; + Self { + integer, + other: part[digits..].to_lowercase(), + } + } +} + +fn compare_version_parts(left: &VersionPart, right: &VersionPart) -> std::cmp::Ordering { + let integer = left.integer.cmp(&right.integer); + if integer != std::cmp::Ordering::Equal { + return integer; + } + match (left.other.is_empty(), right.other.is_empty()) { + (true, true) => std::cmp::Ordering::Equal, + (true, false) => std::cmp::Ordering::Greater, + (false, true) => std::cmp::Ordering::Less, + (false, false) => left.other.encode_utf16().cmp(right.other.encode_utf16()), + } +} + +fn compare_quad_version(left: &str, right: &str) -> Result { + validate_msix_version(left)?; + validate_msix_version(right)?; + let left: Vec = left + .split('.') + .map(|part| part.parse::()) + .collect::>() + .map_err(|_| PublisherError::Invalid("invalid package version".to_owned()))?; + let right: Vec = right + .split('.') + .map(|part| part.parse::()) + .collect::>() + .map_err(|_| PublisherError::Invalid("invalid package version".to_owned()))?; + Ok(left.cmp(&right)) +} + +fn required_string(mapping: &Mapping, key: &str, path: &Path) -> Result { + optional_string(mapping, key) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| PublisherError::Invalid(format!("required field {key} is missing in {}", path.display()))) +} + +fn optional_string(mapping: &Mapping, key: &str) -> Option { + map_get(mapping, key).and_then(|value| value.as_str().map(str::to_owned)) +} + +fn map_get<'a>(mapping: &'a Mapping, key: &str) -> Option<&'a Value> { + mapping.get(Value::String(key.to_owned())) +} + +fn as_mapping(value: &Value) -> Result<&Mapping> { + value + .as_mapping() + .ok_or_else(|| PublisherError::Invalid("manifest document must be a mapping".to_owned())) +} + +fn replace_once(slot: &mut Option, value: Value, label: &str) -> Result<()> { + if slot.replace(value).is_some() { + return Err(PublisherError::Invalid(format!("duplicate {label}"))); + } + Ok(()) +} + +fn merge_yaml_mapping(target: &mut Mapping, source: &Mapping) { + for (key, value) in source { + let is_identity = matches!( + key.as_str(), + Some("ManifestType" | "ManifestVersion" | "PackageIdentifier" | "PackageVersion") + ); + if is_identity && target.contains_key(key) { + continue; + } + match (target.get_mut(key), value) { + (Some(Value::Mapping(existing)), Value::Mapping(incoming)) => { + merge_yaml_mapping(existing, incoming); + } + _ => { + target.insert(key.clone(), value.clone()); + } + } + } +} + +fn set_string(mapping: &mut Mapping, key: &str, value: &str) { + mapping.insert(Value::String(key.to_owned()), Value::String(value.to_owned())); +} + +fn json_quote(value: &str) -> Result { + Ok(serde_json::to_string(value)?) +} + +fn extend_strings(target: &mut BTreeSet, mapping: &Mapping, key: &str) { + match map_get(mapping, key) { + Some(Value::String(value)) => { + target.insert(value.clone()); + } + Some(Value::Sequence(values)) => { + target.extend(values.iter().filter_map(Value::as_str).map(str::to_owned)); + } + _ => {} + } +} + +fn canonicalize_value(value: Value) -> Value { + match value { + Value::Mapping(mapping) => { + let mut entries: Vec<_> = mapping.into_iter().collect(); + entries.sort_by(|left, right| left.0.as_str().unwrap_or("").cmp(right.0.as_str().unwrap_or(""))); + let mut sorted = Mapping::new(); + for (key, value) in entries { + sorted.insert(key, canonicalize_value(value)); + } + Value::Mapping(sorted) + } + Value::Sequence(sequence) => Value::Sequence(sequence.into_iter().map(canonicalize_value).collect()), + other => other, + } +} + +fn gather_files(root: &Path, path: &Path, output: &mut Vec) -> Result<()> { + let metadata = fs::symlink_metadata(path)?; + if is_reparse_point(&metadata) { + return Err(PublisherError::Invalid(format!( + "symlink or reparse point is not allowed: {}", + path.display() + ))); + } + if metadata.is_file() { + output.push(path.to_path_buf()); + } else if metadata.is_dir() { + let mut entries = fs::read_dir(path)?.collect::>>()?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + gather_files(root, &entry.path(), output)?; + } + } + let _ = root; + Ok(()) +} + +fn hash_tree(root: &Path) -> Result> { + let mut files = Vec::new(); + gather_files(root, root, &mut files)?; + files.sort(); + let mut inventory = BTreeMap::new(); + for path in files { + let relative = path + .strip_prefix(root) + .map_err(|_| PublisherError::Invalid(format!("file escaped inventory root: {}", path.display())))?; + let key = relative.to_string_lossy().replace('\\', "/"); + inventory.insert(key, sha256_hex(&fs::read(path)?)); + } + Ok(inventory) +} + +fn reject_reparse_components(path: &Path) -> Result<()> { + let absolute = absolute_normalized(path)?; + let mut current = PathBuf::new(); + for component in absolute.components() { + current.push(component.as_os_str()); + // A bare prefix such as `\\?\D:` names the volume device, not a directory. + if matches!(component, Component::Prefix(_)) { + continue; + } + match fs::symlink_metadata(¤t) { + Ok(metadata) if is_reparse_point(&metadata) => { + return Err(PublisherError::Invalid(format!( + "symbolic links and reparse points are not allowed in publisher paths: {}", + current.display() + ))); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, + Err(error) => return Err(error.into()), + } + } + Ok(()) +} + +fn require_directory(path: &Path) -> Result<()> { + let metadata = fs::symlink_metadata(path)?; + if is_reparse_point(&metadata) || !metadata.is_dir() { + return Err(PublisherError::Invalid(format!( + "expected a regular directory: {}", + path.display() + ))); + } + Ok(()) +} + +fn require_regular_file(path: &Path) -> Result<()> { + let metadata = fs::symlink_metadata(path)?; + if is_reparse_point(&metadata) || !metadata.is_file() { + return Err(PublisherError::Invalid(format!( + "expected a regular file: {}", + path.display() + ))); + } + Ok(()) +} + +fn is_reparse_point(metadata: &fs::Metadata) -> bool { + if metadata.file_type().is_symlink() { + return true; + } + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400; + metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0 + } + #[cfg(not(windows))] + { + false + } +} + +fn validate_build_id(build_id: &str) -> Result<()> { + if !is_lower_hex_hash(build_id) { + return Err(PublisherError::Invalid(format!( + "build identifier must be a lowercase SHA-256 digest: {build_id:?}" + ))); + } + Ok(()) +} + +fn is_lower_hex_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn build_identifier(inventory: &Inventory) -> Result { + let inventory_json = canonical_inventory_json(&inventory.files)?; + let input = format!("pinget-publisher-state-v{PUBLISHER_FORMAT_VERSION}\n{inventory_json}"); + Ok(sha256_hex(input.as_bytes())) +} + +fn canonical_inventory_json(inventory: &BTreeMap) -> Result { + Ok(serde_json::to_string(&CanonicalInventory(inventory))?) +} + +fn serialize_inventory(inventory: &BTreeMap, serializer: S) -> std::result::Result +where + S: Serializer, +{ + CanonicalInventory(inventory).serialize(serializer) +} + +fn compare_utf16(left: &str, right: &str) -> std::cmp::Ordering { + left.encode_utf16().cmp(right.encode_utf16()) +} + +fn absolute_normalized(path: &Path) -> Result { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + other => normalized.push(other.as_os_str()), + } + } + Ok(normalized) +} + +fn canonicalize_future_path(path: &Path) -> Result { + reject_reparse_components(path)?; + let absolute = absolute_normalized(path)?; + if absolute.exists() { + return Ok(fs::canonicalize(absolute)?); + } + + let mut existing = absolute.as_path(); + let mut suffix = Vec::new(); + while !existing.exists() { + let name = existing + .file_name() + .ok_or_else(|| PublisherError::Invalid(format!("no existing parent directory for {}", path.display())))?; + suffix.push(name.to_os_string()); + existing = existing + .parent() + .ok_or_else(|| PublisherError::Invalid(format!("no existing parent directory for {}", path.display())))?; + } + let mut resolved = fs::canonicalize(existing)?; + for component in suffix.into_iter().rev() { + resolved.push(component); + } + Ok(resolved) +} + +fn ensure_disjoint_roots(left: &Path, right: &Path) -> Result<()> { + #[cfg(windows)] + let overlaps = { + let left = windows_path_key(left); + let right = windows_path_key(right); + left == right || left.starts_with(&format!("{right}\\")) || right.starts_with(&format!("{left}\\")) + }; + #[cfg(not(windows))] + let overlaps = left.starts_with(right) || right.starts_with(left); + if overlaps { + return Err(PublisherError::Invalid(format!( + "publisher roots must not overlap: {} and {}", + left.display(), + right.display() + ))); + } + Ok(()) +} + +#[cfg(windows)] +fn windows_path_key(path: &Path) -> String { + let mut value = path.to_string_lossy().replace('/', "\\"); + if let Some(rest) = value.strip_prefix(r"\\?\UNC\") { + value = format!(r"\\{rest}"); + } else if let Some(rest) = value.strip_prefix(r"\\?\") { + value = rest.to_owned(); + } + value.trim_end_matches('\\').to_lowercase() +} + +fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn uuid_string(build_id: &str) -> String { + let digest = Sha256::digest(build_id.as_bytes()); + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[6] = (bytes[6] & 0x0f) | 0x50; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + let hex = bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + format!( + "{}-{}-{}-{}-{}", + &hex[..8], + &hex[8..12], + &hex[12..16], + &hex[16..20], + &hex[20..32] + ) + .to_ascii_uppercase() +} + +#[cfg(windows)] +fn win32_last_error() -> u32 { + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetLastError() -> u32; + } + + // SAFETY: GetLastError reads the calling thread's last-error value and has no pointer arguments. + unsafe { GetLastError() } +} + +#[cfg(windows)] +fn compress_mszip(input: &[u8]) -> Result> { + if input.len() > MAX_MSZIP_SIZE { + return Err(PublisherError::Invalid( + "MSZIP payload exceeds the 64 MiB limit".to_owned(), + )); + } + use std::ffi::c_void; + type CompressorHandle = *mut c_void; + + #[link(name = "cabinet")] + unsafe extern "system" { + fn CreateCompressor(algorithm: u32, options: *const c_void, handle: *mut CompressorHandle) -> i32; + fn Compress( + handle: CompressorHandle, + input: *const c_void, + input_size: usize, + output: *mut c_void, + output_size: usize, + bytes_written: *mut usize, + ) -> i32; + fn CloseCompressor(handle: CompressorHandle) -> i32; + } + + const COMPRESS_ALGORITHM_MSZIP: u32 = 2; + const ERROR_INSUFFICIENT_BUFFER: u32 = 122; + let mut handle = std::ptr::null_mut(); + // SAFETY: API receives a valid output pointer and null optional options. + if unsafe { CreateCompressor(COMPRESS_ALGORITHM_MSZIP, std::ptr::null(), &mut handle) } == 0 { + return Err(PublisherError::Tool(format!( + "CreateCompressor failed: {}", + win32_last_error() + ))); + } + let result = (|| { + let mut required = 0usize; + // SAFETY: query mode uses null output and a valid input slice. + let success = unsafe { + Compress( + handle, + input.as_ptr().cast(), + input.len(), + std::ptr::null_mut(), + 0, + &mut required, + ) + }; + if success == 0 && win32_last_error() != ERROR_INSUFFICIENT_BUFFER { + return Err(PublisherError::Tool(format!( + "MSZIP size query failed: {}", + win32_last_error() + ))); + } + let mut output = vec![0u8; required]; + let mut written = 0usize; + // SAFETY: output has the API-reported capacity and both slices remain alive. + if unsafe { + Compress( + handle, + input.as_ptr().cast(), + input.len(), + output.as_mut_ptr().cast(), + output.len(), + &mut written, + ) + } == 0 + { + return Err(PublisherError::Tool(format!( + "MSZIP compression failed: {}", + win32_last_error() + ))); + } + output.truncate(written); + Ok(output) + })(); + // SAFETY: handle was successfully created and is closed exactly once. + unsafe { + CloseCompressor(handle); + } + result +} + +#[cfg(windows)] +fn decompress_mszip(input: &[u8]) -> Result> { + validate_mszip_size(input)?; + use std::ffi::c_void; + type DecompressorHandle = *mut c_void; + + #[link(name = "cabinet")] + unsafe extern "system" { + fn CreateDecompressor(algorithm: u32, options: *const c_void, handle: *mut DecompressorHandle) -> i32; + fn Decompress( + handle: DecompressorHandle, + input: *const c_void, + input_size: usize, + output: *mut c_void, + output_size: usize, + bytes_written: *mut usize, + ) -> i32; + fn CloseDecompressor(handle: DecompressorHandle) -> i32; + } + + const COMPRESS_ALGORITHM_MSZIP: u32 = 2; + const ERROR_INSUFFICIENT_BUFFER: u32 = 122; + let mut handle = std::ptr::null_mut(); + // SAFETY: API receives a valid output pointer and null optional options. + if unsafe { CreateDecompressor(COMPRESS_ALGORITHM_MSZIP, std::ptr::null(), &mut handle) } == 0 { + return Err(PublisherError::Tool(format!( + "CreateDecompressor failed: {}", + win32_last_error() + ))); + } + let result = (|| { + let mut required = 0usize; + // SAFETY: query mode uses null output and a valid input slice. + let success = unsafe { + Decompress( + handle, + input.as_ptr().cast(), + input.len(), + std::ptr::null_mut(), + 0, + &mut required, + ) + }; + if success == 0 && win32_last_error() != ERROR_INSUFFICIENT_BUFFER { + return Err(PublisherError::Tool(format!( + "MSZIP size query failed: {}", + win32_last_error() + ))); + } + if required > MAX_MSZIP_SIZE { + return Err(PublisherError::Invalid( + "MSZIP decoded payload exceeds the 64 MiB limit".to_owned(), + )); + } + let mut output = vec![0u8; required]; + let mut written = 0usize; + // SAFETY: output has the API-reported capacity and both slices remain alive. + if unsafe { + Decompress( + handle, + input.as_ptr().cast(), + input.len(), + output.as_mut_ptr().cast(), + output.len(), + &mut written, + ) + } == 0 + { + return Err(PublisherError::Tool(format!( + "MSZIP decompression failed: {}", + win32_last_error() + ))); + } + output.truncate(written); + Ok(output) + })(); + // SAFETY: handle was successfully created and is closed exactly once. + unsafe { + CloseDecompressor(handle); + } + result +} + +fn validate_mszip_size(input: &[u8]) -> Result<()> { + let size = input + .get(8..16) + .and_then(|bytes| <[u8; 8]>::try_from(bytes).ok()) + .map(u64::from_le_bytes) + .ok_or_else(|| PublisherError::Invalid("truncated MSZIP header".to_owned()))?; + if size > MAX_MSZIP_SIZE as u64 { + return Err(PublisherError::Invalid( + "MSZIP decoded payload exceeds the 64 MiB limit".to_owned(), + )); + } + Ok(()) +} + +#[cfg(not(windows))] +fn compress_mszip(_input: &[u8]) -> Result> { + Err(PublisherError::Tool( + "native WinGet MSZIP compression is currently supported only on Windows".to_owned(), + )) +} + +#[cfg(not(windows))] +fn decompress_mszip(input: &[u8]) -> Result> { + validate_mszip_size(input)?; + Err(PublisherError::Tool( + "native WinGet MSZIP decompression is currently supported only on Windows".to_owned(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn version_comparer_orders_numeric_release_and_special_versions() { + assert_eq!(compare_winget_versions("1.10.0", "1.9.0"), std::cmp::Ordering::Greater); + assert_eq!(compare_winget_versions("1.2.0", "1.2"), std::cmp::Ordering::Equal); + assert_eq!(compare_winget_versions("1.2-rc1", "1.2"), std::cmp::Ordering::Less); + assert_eq!(compare_winget_versions("< 1.2", "1.2"), std::cmp::Ordering::Less); + assert_eq!(compare_winget_versions("> 1.2", "1.2"), std::cmp::Ordering::Greater); + assert_eq!(compare_winget_versions("unknown", "0.1"), std::cmp::Ordering::Less); + assert_eq!(compare_winget_versions("latest", "99.0"), std::cmp::Ordering::Greater); + } + + #[test] + fn schema_scalars_preserve_raw_spelling_in_block_and_flow_yaml() { + let hash = "1".repeat(64); + let input = format!( + "PackageVersion: 1.0\nUnknown: 2024\nFlag: true\nTags: [1.0, 2024, true]\nInstallers:\n- InstallerSha256: {hash}\n SignatureSha256: {hash}\n Details: {{MinimumVersion: 2024, DisplayVersion: 1.0}}\nDescription: |\n InstallerSha256: 123\n" + ); + let output = serde::de::DeserializeSeed::deserialize(ManifestValue, serde_yaml::Deserializer::from_str(&input)) + .expect("schema-aware YAML"); + let map = output.as_mapping().expect("mapping"); + assert_eq!(optional_string(map, "PackageVersion").as_deref(), Some("1.0")); + assert_eq!(map_get(map, "Unknown").and_then(Value::as_u64), Some(2024)); + assert_eq!(map_get(map, "Flag").and_then(Value::as_bool), Some(true)); + assert_eq!( + map_get(map, "Tags").and_then(Value::as_sequence).expect("tags"), + &["1.0", "2024", "true"].map(|value| Value::String(value.to_owned())) + ); + let installer = map_get(map, "Installers") + .and_then(Value::as_sequence) + .expect("installers")[0] + .as_mapping() + .expect("installer"); + assert_eq!( + optional_string(installer, "InstallerSha256").as_deref(), + Some(hash.as_str()) + ); + assert_eq!( + optional_string(installer, "SignatureSha256").as_deref(), + Some(hash.as_str()) + ); + let details = map_get(installer, "Details") + .and_then(Value::as_mapping) + .expect("details"); + assert_eq!(optional_string(details, "MinimumVersion").as_deref(), Some("2024")); + assert_eq!(optional_string(details, "DisplayVersion").as_deref(), Some("1.0")); + assert_eq!( + optional_string(map, "Description").as_deref(), + Some("InstallerSha256: 123\n") + ); + } + + #[test] + fn catalog_terms_use_ascii_nocase_with_ordinal_spelling() { + let terms = ["tool", "TOOL", "Tool", "", " ", "é", "É"] + .map(str::to_owned) + .into_iter() + .collect(); + assert_eq!(canonical_terms(terms), vec!["TOOL", "É", "é"]); + } + + #[test] + fn unknown_numeric_scalars_keep_number_types_and_canonical_text() { + let yaml = "[1.0, 0.5, 1e-5, 1e-6, 1e15, 1e16, 1e20, -0.0, 18446744073709551615]"; + let value = serde::de::DeserializeSeed::deserialize(ManifestValue, serde_yaml::Deserializer::from_str(yaml)) + .expect("numbers"); + let json = serde_json::to_string(&CanonicalYaml(&value)).expect("canonical numbers"); + assert_eq!( + json, + "[1.0,0.5,0.00001,1e-6,1000000000000000.0,1e+16,1e+20,-0.0,18446744073709551615]" + ); + } + + #[test] + fn mszip_declared_size_limit_is_checked_before_allocation() { + let mut header = [0u8; 24]; + header[8..16].copy_from_slice(&(MAX_MSZIP_SIZE as u64).to_le_bytes()); + assert!(validate_mszip_size(&header).is_ok(), "64 MiB boundary is supported"); + header[8..16].copy_from_slice(&((MAX_MSZIP_SIZE as u64) + 1).to_le_bytes()); + assert!( + validate_mszip_size(&header).is_err(), + "over limit must fail without decompression" + ); + assert!( + validate_mszip_size(&header[..12]).is_err(), + "truncated header must fail" + ); + } + + fn unit_directory(label: &str) -> PathBuf { + let name = format!( + ".publisher-unit-{label}-{}-{}", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_nanos() + ); + let directory = std::env::current_dir().expect("cwd").join(name); + fs::create_dir(&directory).expect("create unit directory"); + directory + } + + #[test] + fn future_or_unrelated_state_is_never_initialized_or_mutated() { + for future in [false, true] { + let directory = unit_directory("state"); + let path = directory.join("publisher.db"); + let connection = Connection::open(&path).expect("create database"); + connection + .execute_batch("CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES('keep');") + .expect("schema"); + if future { + connection + .execute_batch("PRAGMA user_version=42") + .expect("future version"); + } + + drop(connection); + let original = fs::read(&path).expect("database"); + assert!(open_state(&directory, true).is_err(), "foreign state must be rejected"); + assert_eq!( + fs::read(&path).expect("database after"), + original, + "database bytes must remain unchanged" + ); + fs::remove_dir_all(directory).expect("cleanup"); + } + } + + #[test] + fn legacy_artifact_format_requires_fresh_state_without_migration() { + let root = unit_directory("legacy"); + let connection = open_state(&root, true).expect("current state"); + connection + .execute("DELETE FROM metadata WHERE name='artifactFormatVersion'", []) + .expect("legacy metadata"); + drop(connection); + let database = fs::read(root.join("publisher.db")).expect("legacy state"); + fs::write(root.join("old-snapshot"), b"preserve").expect("prior artifact"); + let error = open_state(&root, true).expect_err("legacy format must be refused"); + assert!( + error.to_string().contains("new --state"), + "actionable fresh-state guidance" + ); + assert_eq!( + fs::read(root.join("publisher.db")).expect("unchanged database"), + database + ); + assert_eq!( + fs::read(root.join("old-snapshot")).expect("prior artifact"), + b"preserve" + ); + let inventory = Inventory { + files: BTreeMap::new(), + manifests: Vec::new(), + }; + assert_eq!( + build_identifier(&inventory).expect("identifier"), + sha256_hex(b"pinget-publisher-state-v2\n{}") + ); + assert_ne!( + build_identifier(&inventory).expect("identifier"), + sha256_hex(b"pinget-publisher-state-v1\n{}") + ); + fs::remove_dir_all(root).expect("cleanup"); + } + + #[test] + fn owned_staging_recovery_leaves_sibling_artifacts_untouched() { + let root = unit_directory("staging"); + let staging = root.join(format!(".staging-{}", "1".repeat(64))); + fs::create_dir_all(staging.join("feed")).expect("stale staging"); + fs::write(staging.join("feed").join("partial"), b"partial").expect("partial artifact"); + fs::write(root.join("keep"), b"immutable").expect("sibling"); + remove_owned_staging(&staging).expect("recover owned staging"); + assert!(!staging.exists()); + assert_eq!(fs::read(root.join("keep")).expect("sibling"), b"immutable"); + fs::remove_dir_all(root).expect("cleanup"); + } + + #[test] + fn failed_publication_removes_only_new_files_and_empty_directories() { + let root = unit_directory("rollback"); + let existing = root.join("existing"); + fs::create_dir(&existing).expect("existing directory"); + fs::write(existing.join("keep"), b"old").expect("existing artifact"); + { + let mut rollback = PublicationRollback::default(); + let directory = root.join("new").join("nested"); + rollback + .create_directories(&root, &directory) + .expect("created directories"); + let path = directory.join("artifact"); + rollback.files.push(path.clone()); + fs::write(path, b"new").expect("new artifact"); + } + assert!(!root.join("new").exists(), "new directory tree must be removed"); + assert_eq!(fs::read(existing.join("keep")).expect("existing artifact"), b"old"); + fs::remove_dir_all(root).expect("cleanup"); + } + + #[test] + fn destination_lock_serializes_independent_publisher_states() { + let root = unit_directory("lock"); + let destination = root.join("feed"); + let first = acquire_destination_lock(&destination).expect("first lock"); + assert!( + acquire_destination_lock(&destination).is_err(), + "same destination must be exclusive" + ); + assert!(!destination.exists(), "locking must not create a feed"); + drop(first); + drop(acquire_destination_lock(&destination).expect("released lock")); + fs::remove_dir_all(root).expect("cleanup"); + } + + #[test] + fn sdk_discovery_prefers_numeric_newest_version_and_host_architecture() { + let root = unit_directory("sdk"); + for (version, architecture) in [("10.0.9.0", "x64"), ("10.0.10.0", "x86"), ("10.0.10.0", "x64")] { + let directory = root.join(version).join(architecture); + fs::create_dir_all(&directory).expect("SDK directory"); + fs::write(directory.join("makeappx.exe"), b"fixture").expect("SDK tool"); + } + assert_eq!( + find_sdk_tool(&root, "makeappx.exe", &["x64", "x86"]), + Some(root.join("10.0.10.0").join("x64").join("makeappx.exe")) + ); + fs::remove_file(root.join("10.0.10.0").join("x64").join("makeappx.exe")).expect("remove native tool"); + assert_eq!( + find_sdk_tool(&root, "makeappx.exe", &["x64", "x86"]), + Some(root.join("10.0.10.0").join("x86").join("makeappx.exe")) + ); + assert!(find_sdk_tool(&root, "signtool.exe", &["x64"]).is_none()); + fs::remove_dir_all(root).expect("cleanup"); + } + + #[test] + fn recorded_snapshot_requires_matching_input_inventory_and_summary() { + let connection = Connection::open_in_memory().expect("state"); + connection.execute_batch(STATE_SCHEMA).expect("schema"); + let mut snapshot = SnapshotMetadata { + schema_version: 1, + artifact_format_version: PUBLISHER_FORMAT_VERSION, + build_id: "1".repeat(64), + created_utc: "now".to_owned(), + manifest_count: 1, + package_count: 1, + index_hash: "2".repeat(64), + inventory: [("input.yaml".to_owned(), "3".repeat(64))].into_iter().collect(), + artifacts: BTreeMap::new(), + }; + assert!( + validate_recorded_snapshot(&connection, &snapshot).is_err(), + "orphan snapshot rejected" + ); + connection + .execute( + "INSERT INTO builds(id,inventory,summary) VALUES(?1,?2,?3)", + params![ + snapshot.build_id, + canonical_inventory_json(&snapshot.inventory).expect("inventory"), + r#"{"createdUtc":"now","manifestCount":1,"packageCount":1}"# + ], + ) + .expect("record"); + validate_recorded_snapshot(&connection, &snapshot).expect("matching snapshot"); + snapshot.inventory.insert("other.yaml".to_owned(), "4".repeat(64)); + assert!( + validate_recorded_snapshot(&connection, &snapshot).is_err(), + "different inventory rejected" + ); + snapshot.inventory.remove("other.yaml"); + snapshot.package_count = 2; + assert!( + validate_recorded_snapshot(&connection, &snapshot).is_err(), + "different summary rejected" + ); + } + + #[test] + fn shared_tag_rows_use_nocase_and_minimum_spelling_across_packages() { + let connection = Connection::open_in_memory().expect("index"); + connection.execute_batch(INDEX_SCHEMA).expect("schema"); + insert_terms(&connection, "tags2", "tag", "tags2_map", "tag", 1, &["tool".to_owned()]).expect("first tag"); + insert_terms(&connection, "tags2", "tag", "tags2_map", "tag", 2, &["TOOL".to_owned()]).expect("second tag"); + let tags: Vec = connection + .prepare("SELECT tag FROM tags2") + .expect("query") + .query_map([], |row| row.get(0)) + .expect("rows") + .collect::>() + .expect("tags"); + assert_eq!(tags, vec!["TOOL"]); + let count: i64 = connection + .query_row("SELECT COUNT(*) FROM tags2_map", [], |row| row.get(0)) + .expect("maps"); + assert_eq!(count, 2, "both packages retain the same shared row"); + } +} diff --git a/rust/crates/pinget-publisher/src/state-schema.sql b/rust/crates/pinget-publisher/src/state-schema.sql new file mode 100644 index 0000000..543fa0d --- /dev/null +++ b/rust/crates/pinget-publisher/src/state-schema.sql @@ -0,0 +1,10 @@ +PRAGMA user_version=1; +CREATE TABLE metadata( + name TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL +) WITHOUT ROWID; +CREATE TABLE builds( + id TEXT PRIMARY KEY NOT NULL, + inventory TEXT NOT NULL, + summary TEXT NOT NULL +) WITHOUT ROWID; diff --git a/rust/crates/pinget-publisher/tests/conformance.rs b/rust/crates/pinget-publisher/tests/conformance.rs new file mode 100644 index 0000000..95351d9 --- /dev/null +++ b/rust/crates/pinget-publisher/tests/conformance.rs @@ -0,0 +1,1594 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use pinget_publisher::{ + BuildOptions, PublishOptions, Signing, build, diff, doctor, publish, status, verify, verify_published_destination, +}; +use rusqlite::Connection; +use serde_json::Value; + +static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +#[test] +fn build_multifile_manifest_set_emits_complete_v2_index_and_feed() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("multi-file fixture should build"); + + assert!(!report.build_id.is_empty()); + assert!(verify(&state, Some(&report.build_id), None).is_ok()); + let latest_build_id = status(&state) + .expect("status should reopen the newly built state") + .last_build + .map(|latest| latest.build_id); + assert_eq!(latest_build_id.as_deref(), Some(report.build_id.as_str())); + + let snapshot = state.join("builds").join(&report.build_id); + let index = snapshot.join("Public").join("index.db"); + let feed = snapshot.join("feed"); + assert!(index.is_file()); + let metadata: Value = serde_json::from_slice( + &fs::read(snapshot.join("snapshot.json")).expect("snapshot metadata should be readable"), + ) + .expect("snapshot metadata should be JSON"); + assert_eq!(metadata["schemaVersion"].as_u64(), Some(1)); + assert_eq!(metadata["artifactFormatVersion"].as_u64(), Some(2)); + let inventory_json = + serde_json::to_string(&metadata["inventory"]).expect("snapshot inventory should have canonical JSON"); + let expected_build_id = sha256_hex(format!("pinget-publisher-state-v2\n{inventory_json}").as_bytes()); + let legacy_build_id = sha256_hex(format!("pinget-publisher-state-v1\n{inventory_json}").as_bytes()); + assert_eq!(report.build_id, expected_build_id); + assert_ne!(report.build_id, legacy_build_id); + let state_database = Connection::open(state.join("publisher.db")).expect("publisher state database should open"); + let state_artifact_format: String = state_database + .query_row( + "SELECT value FROM metadata WHERE name='artifactFormatVersion'", + [], + |row| row.get(0), + ) + .expect("publisher artifact format marker should exist"); + let state_schema_version: String = state_database + .query_row("SELECT value FROM metadata WHERE name='schemaVersion'", [], |row| { + row.get(0) + }) + .expect("publisher schema marker should exist"); + let state_user_version: i64 = state_database + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .expect("publisher SQLite user_version should be readable"); + assert_eq!(state_artifact_format, "2"); + assert_eq!(state_schema_version, "1"); + assert_eq!(state_user_version, 1); + let expected_index_hash = sha256_file(&index); + assert_eq!( + metadata["indexHash"].as_str(), + Some(expected_index_hash.as_str()), + "snapshot metadata must bind the exact v2 index bytes" + ); + let database = Connection::open_with_flags(&index, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("SQLite index should open independently"); + let package: (String, String, String, Vec) = database + .query_row("SELECT id,name,latest_version,hash FROM packages", [], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) + }) + .expect("index should contain the package"); + assert_eq!(package.0, "Pinget.Test.Matrix"); + assert_eq!(package.1, "Matrix Package"); + assert_eq!(package.2, "1.10.0"); + assert_eq!(package.3.len(), 32); + + assert_eq!( + read_mapped_values(&database, "tags2", "tag", "tags2_map", "tag"), + ["Matrix", "matrice", "publisher-conformance"] + ); + assert_eq!( + read_mapped_values(&database, "commands2", "command", "commands2_map", "command"), + ["Matrix", "OtherCmd", "matrix-cli"] + ); + let arp_bounds: (String, String) = database + .query_row( + "SELECT arp_min_version,arp_max_version FROM packages WHERE id=?1", + ["Pinget.Test.Matrix"], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("index should contain ARP version bounds"); + assert_eq!(arp_bounds, ("1.9.0".to_owned(), "1.10.0".to_owned())); + + for (table, column, expected) in [ + ("pfns2", "pfn", "Pinget.Matrix_1234567890abc"), + ("productcodes2", "productcode", "{00112233-4455-6677-8899-AABBCCDDEEFF}"), + ("upgradecodes2", "upgradecode", "{11223344-5566-7788-99AA-BBCCDDEEFF00}"), + ] { + assert_eq!( + read_correlations(&database, table, column), + [expected], + "{table} mapping" + ); + } + assert_eq!( + read_correlations(&database, "norm_names2", "norm_name"), + ["matrice", "matrixdesktop", "matrixpackage"] + ); + assert_eq!( + read_correlations(&database, "norm_publishers2", "norm_publisher"), + ["pingetlocalizationlab", "pingettestfixtures"] + ); + + let sidecar_hash = encode_hex(&package.3); + let sidecar = feed + .join("packages") + .join("Pinget.Test.Matrix") + .join(&sidecar_hash[..8]) + .join("versionData.mszyml"); + assert!(sidecar.is_file()); + assert_eq!(sha256_file(&sidecar), sidecar_hash); + let hosted_manifest_count = count_files_with_extension(&feed.join("manifests"), "yaml"); + assert_eq!( + hosted_manifest_count, 2, + "one hosted manifest should be emitted per version" + ); + + let locale_directory = feed.join("manifests/Pinget.Test.Matrix/1.10.0"); + let manifest_path = find_first_file(&locale_directory); + let manifest: Value = serde_json::from_slice(&fs::read(manifest_path).expect("merged manifest should be readable")) + .expect("merged manifest should be JSON"); + let installers = manifest["Installers"] + .as_array() + .expect("merged manifest should retain installer entries"); + assert_eq!( + installers[0]["InstallerSha256"].as_str(), + Some("1111111111111111111111111111111111111111111111111111111111111111"), + "all-digit InstallerSha256 scalars are schema strings, not numbers" + ); + assert_eq!( + installers[1]["InstallerSha256"].as_str(), + Some("2222222222222222222222222222222222222222222222222222222222222222"), + "all-digit InstallerSha256 scalars are schema strings, not numbers" + ); + let older_locale_directory = feed.join("manifests/Pinget.Test.Matrix/1.9.0"); + let older_manifest_path = find_first_file(&older_locale_directory); + let older_manifest: Value = + serde_json::from_slice(&fs::read(older_manifest_path).expect("older merged manifest should be readable")) + .expect("older merged manifest should be JSON"); + assert_eq!( + older_manifest["Installers"][0]["InstallerSha256"].as_str(), + Some("0000000000000000000000000000000000000000000000000000000000000000"), + "all-zero InstallerSha256 must not be coerced to numeric zero" + ); + + let current_manifest_path = find_first_file(&feed.join("manifests/Pinget.Test.Matrix/1.10.0")); + let current_manifest: Value = + serde_json::from_slice(&fs::read(current_manifest_path).expect("merged manifest should be readable")) + .expect("merged manifest should be JSON"); + assert_eq!(current_manifest["PackageLocale"].as_str(), Some("en-US")); + assert_eq!(current_manifest["PackageName"].as_str(), Some("Matrix Package")); + assert_eq!(current_manifest["Publisher"].as_str(), Some("Pinget Test Fixtures")); + assert_eq!(current_manifest["PackageVersion"].as_str(), Some("1.10.0")); + let french_localization = current_manifest["Localization"] + .as_array() + .expect("non-default locale should be embedded in Localization"); + assert_eq!(french_localization.len(), 1); + assert_eq!(french_localization[0]["PackageLocale"].as_str(), Some("fr-FR")); + assert_eq!(french_localization[0]["PackageName"].as_str(), Some("Matrice")); + assert_eq!( + french_localization[0]["Publisher"].as_str(), + Some("Pinget Localization Lab") + ); + assert!(french_localization[0].get("ManifestType").is_none()); + let installers = current_manifest["Installers"] + .as_array() + .expect("French merged manifest should retain installer entries"); + assert_eq!(installers.len(), 2); + assert_eq!( + current_manifest["Installers"][0]["Dependencies"]["PackageDependencies"][0]["PackageIdentifier"].as_str(), + Some("Pinget.Test.Runtime") + ); + assert_eq!( + current_manifest["Installers"][0]["InstallerSwitches"]["Silent"].as_str(), + Some("/quiet") + ); + assert!( + installers + .iter() + .any(|installer| installer["InstallerType"].as_str() == Some("portable")) + ); +} + +#[test] +fn build_uses_declared_default_locale_and_embeds_other_localizations_once() { + let temp = TempDirectory::new(); + let input = temp.path.join("manifests"); + fs::create_dir_all(&input).expect("manifest root should be created"); + let id = "Pinget.Test.LocaleOrder"; + write_locale_order_manifests(&input, id); + + let report = build(&BuildOptions { + input, + state: temp.path.join("state"), + }) + .expect("a default locale that sorts after another locale should build"); + + let feed_manifests = report.snapshot_directory.join("feed/manifests").join(id).join("1.0.0"); + assert_eq!( + count_files_with_extension(&feed_manifests, "yaml"), + 1, + "one merged manifest should be emitted per version" + ); + let merged: serde_yaml::Value = serde_yaml::from_slice( + &fs::read(find_first_file(&feed_manifests)).expect("merged manifest should be readable"), + ) + .expect("merged manifest should parse as YAML"); + assert_eq!(merged["PackageLocale"].as_str(), Some("en-US")); + assert_eq!(merged["PackageName"].as_str(), Some("Locale Order English")); + let localizations = merged["Localization"] + .as_sequence() + .expect("non-default locales should be embedded in Localization"); + assert_eq!(localizations.len(), 1); + assert_eq!(localizations[0]["PackageLocale"].as_str(), Some("de-DE")); + assert_eq!(localizations[0]["PackageName"].as_str(), Some("Locale Reihenfolge")); + assert!(localizations[0].get("ManifestType").is_none()); + assert!(localizations[0].get("ManifestVersion").is_none()); + + let database = Connection::open_with_flags( + report.snapshot_directory.join("Public/index.db"), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .expect("index should open read-only"); + let indexed_versions: Vec = database + .prepare("SELECT latest_version FROM packages WHERE id=?1") + .expect("package query should prepare") + .query_map([id], |row| row.get(0)) + .expect("package query should run") + .map(|row| row.expect("version row should be readable")) + .collect(); + assert_eq!(indexed_versions, ["1.0.0"]); +} + +#[test] +fn build_preserves_schema_string_scalars_in_canonical_manifest_output() { + let temp = TempDirectory::new(); + let input = temp.path.join("manifests"); + fs::create_dir_all(&input).expect("manifest root should be created"); + fs::write( + input.join("scalar.yaml"), + r#"PackageIdentifier: Pinget.Test.ScalarTypes +PackageVersion: 1.0 +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PackageName: Scalar Types +DisplayVersion: 1.0 +SignatureSha256: 0000000000000000000000000000000000000000000000000000000000000000 +GenericNumber: 1.0 +Small: 0.5 +Tiny: 1e-6 +Threshold: 1e-5 +Large: 1e16 +NegativeZero: -0.0 +MaxUInt: 18446744073709551615 +Installers: + - InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 + Architecture: x64 + InstallerType: exe + InstallerUrl: https://example.invalid/scalar.exe +ManifestType: singleton +ManifestVersion: 1.9.0 +"#, + ) + .expect("scalar fixture should be written"); + + let report = build(&BuildOptions { + input, + state: temp.path.join("state"), + }) + .expect("valid schema scalar fixture should build"); + + let merged_path = find_first_file(&report.snapshot_directory.join("feed/manifests")); + let merged: serde_yaml::Value = + serde_yaml::from_slice(&fs::read(merged_path).expect("merged output should be readable")) + .expect("merged output should parse as YAML"); + assert_eq!(merged["PackageVersion"].as_str(), Some("1.0")); + assert_eq!(merged["DisplayVersion"].as_str(), Some("1.0")); + assert_eq!( + merged["SignatureSha256"].as_str(), + Some("0000000000000000000000000000000000000000000000000000000000000000") + ); + assert_eq!( + merged["Installers"][0]["InstallerSha256"].as_str(), + Some("0000000000000000000000000000000000000000000000000000000000000000") + ); + assert_eq!(merged["GenericNumber"].as_f64(), Some(1.0)); + assert_eq!(merged["Small"].as_f64(), Some(0.5)); + assert_eq!(merged["Tiny"].as_f64(), Some(1e-6)); + assert_eq!(merged["Threshold"].as_f64(), Some(1e-5)); + assert_eq!(merged["Large"].as_f64(), Some(1e16)); + assert_eq!(merged["NegativeZero"].as_f64(), Some(-0.0)); + assert_eq!(merged["MaxUInt"].as_u64(), Some(u64::MAX)); +} + +#[test] +fn equivalent_input_trees_produce_the_same_build_identifier() { + let temp = TempDirectory::new(); + let first_input = copy_fixture("multi", &temp.path.join("first")); + let second_input = copy_fixture("multi", &temp.path.join("second")); + let first = build(&BuildOptions { + input: first_input, + state: temp.path.join("state-a"), + }) + .expect("first input should build"); + let second = build(&BuildOptions { + input: second_input, + state: temp.path.join("state-b"), + }) + .expect("second input should build"); + + assert_eq!(first.build_id, second.build_id); +} + +#[test] +fn diff_reports_raw_content_additions_changes_and_removals() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("initial fixture should build"); + let unchanged = diff(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("unchanged inputs should diff"); + assert!(unchanged.added.is_empty()); + assert!(unchanged.changed.is_empty()); + assert!(unchanged.removed.is_empty()); + + let changed_manifest = input.join("Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + let mut changed_content = fs::read_to_string(&changed_manifest).expect("fixture should be readable"); + changed_content.push_str("# raw byte update\n"); + fs::write(changed_manifest, changed_content).expect("metadata file should change"); + fs::write( + input.join("Pinget.Test.Added.yaml"), + singleton_manifest("Pinget.Test.Added", "1.0.0"), + ) + .expect("new manifest should be written"); + fs::remove_file(input.join("Pinget.Test.Matrix_1.9.0.yaml")).expect("old version fixture should be removed"); + + let changes = diff(&BuildOptions { input, state }).expect("changed input should diff"); + assert_eq!(changes.added, ["Pinget.Test.Added.yaml"]); + assert_eq!(changes.changed, ["Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"]); + assert_eq!(changes.removed, ["Pinget.Test.Matrix_1.9.0.yaml"]); + assert!(!changes.last_build_published); +} + +#[test] +fn readonly_state_commands_do_not_initialize_missing_state() { + let temp = TempDirectory::new(); + let missing_state = temp.path.join("missing"); + assert!(status(&missing_state).is_err()); + assert!(!missing_state.exists()); + + let input = copy_fixture("multi", &temp.path.join("manifests")); + assert!( + diff(&BuildOptions { + input, + state: missing_state.clone(), + }) + .is_err() + ); + assert!(!missing_state.exists()); +} + +#[test] +fn read_only_operations_reject_reparse_points_and_state_overlap() { + #[cfg(windows)] + { + use std::os::windows::fs::symlink_dir; + + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let report = build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("fixture should build before reparse checks"); + let input_link = temp.path.join("manifests-link"); + let state_link = temp.path.join("state-link"); + symlink_dir(&input, &input_link).expect("input directory symlink should be available"); + symlink_dir(&state, &state_link).expect("state directory symlink should be available"); + let diff_options = BuildOptions { + input: input_link, + state, + }; + assert!(diff(&diff_options).is_err()); + assert!(status(&state_link).is_err()); + assert!(verify(&state_link, Some(&report.build_id), None).is_err()); + fs::remove_dir(&diff_options.input).expect("input symlink should be removed"); + fs::remove_dir(state_link).expect("state symlink should be removed"); + } + + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("fixture should build before overlap check"); + let nested_input = state.join("builds").join(&report.build_id).join("feed"); + let error = diff(&BuildOptions { + input: nested_input, + state: state.clone(), + }) + .expect_err("diff must reject an input root nested inside its state root"); + assert!(error.to_string().to_ascii_lowercase().contains("overlap")); + assert!(verify(&state, Some(&report.build_id), None).is_ok()); +} + +#[test] +fn build_rejects_future_sqlite_user_version_without_mutating_state() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("initial fixture should create state"); + let database_path = state.join("publisher.db"); + { + let database = Connection::open(&database_path).expect("state database should open"); + database + .execute_batch("PRAGMA user_version=999;") + .expect("future schema marker should be written"); + } + let original_database = fs::read(&database_path).expect("future-version database should be readable"); + + let error = build(&BuildOptions { input, state }).expect_err("a future user_version must be rejected"); + + assert!(error.to_string().to_ascii_lowercase().contains("version")); + assert_eq!( + fs::read(database_path).expect("database should remain readable"), + original_database, + "rejecting an unknown state version must not rewrite the database" + ); +} + +#[test] +fn build_rejects_legacy_artifact_format_state_without_mutating_it() { + for format_version in [None, Some("1"), Some("999")] { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("current-format fixture should create state"); + let database_path = state.join("publisher.db"); + { + let database = Connection::open(&database_path).expect("state database should open"); + if let Some(version) = format_version { + database + .execute( + "UPDATE metadata SET value=?1 WHERE name='artifactFormatVersion'", + [version], + ) + .expect("unsupported format marker should be written"); + } else { + database + .execute("DELETE FROM metadata WHERE name='artifactFormatVersion'", []) + .expect("legacy format marker should be removed"); + } + } + let original_database = fs::read(&database_path).expect("legacy-format database should be readable"); + + let error = build(&BuildOptions { input, state }).expect_err("unsupported publisher state must be rejected"); + + let diagnostic = error.to_string().to_ascii_lowercase(); + assert!( + diagnostic.contains("artifact format version"), + "version: {format_version:?}; {diagnostic}" + ); + assert!( + diagnostic.contains("--state"), + "version: {format_version:?}; {diagnostic}" + ); + assert_eq!( + fs::read(database_path).expect("database should remain readable"), + original_database, + "refusing format {format_version:?} must not migrate or rewrite it" + ); + } +} + +#[test] +fn verify_rejects_legacy_artifact_format_snapshot_without_mutating_it() { + for format_version in [None, Some(1), Some(999)] { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("current-format fixture should create a snapshot"); + let metadata_path = state.join("builds").join(&report.build_id).join("snapshot.json"); + let mut metadata: Value = + serde_json::from_slice(&fs::read(&metadata_path).expect("snapshot should be readable")) + .expect("snapshot metadata should be JSON"); + assert_eq!(metadata["artifactFormatVersion"].as_u64(), Some(2)); + if let Some(version) = format_version { + metadata["artifactFormatVersion"] = Value::from(version); + } else { + metadata + .as_object_mut() + .expect("snapshot metadata should be an object") + .remove("artifactFormatVersion"); + } + fs::write( + &metadata_path, + serde_json::to_vec_pretty(&metadata).expect("legacy metadata should serialize"), + ) + .expect("legacy snapshot metadata should be written"); + let legacy_snapshot = fs::read(&metadata_path).expect("legacy snapshot should be readable"); + + let error = + verify(&state, Some(&report.build_id), None).expect_err("unsupported publisher snapshot must be rejected"); + + assert!(error.to_string().to_ascii_lowercase().contains("format")); + assert_eq!( + fs::read(&metadata_path).expect("snapshot should remain readable"), + legacy_snapshot, + "verification must not upgrade format {format_version:?} snapshot metadata" + ); + } +} + +#[test] +fn build_rejects_foreign_sqlite_state_without_mutating_it() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + fs::create_dir_all(&state).expect("state root should be created"); + let database_path = state.join("publisher.db"); + { + let database = Connection::open(&database_path).expect("foreign database should be created"); + database + .execute_batch( + "CREATE TABLE unrelated_owner(value TEXT); INSERT INTO unrelated_owner VALUES('preserve-me');", + ) + .expect("foreign schema should be written"); + } + let original_database = fs::read(&database_path).expect("foreign database should be readable"); + + let error = build(&BuildOptions { input, state }).expect_err("unrecognized SQLite state must not be adopted"); + + assert!(!error.to_string().is_empty()); + assert_eq!( + fs::read(database_path).expect("foreign database should remain readable"), + original_database + ); +} + +#[test] +fn build_preserves_stale_staging_owned_by_another_attempt_and_completes_snapshot() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let stale_staging = state.join("builds/.staging-stale-owned-attempt"); + fs::create_dir_all(&stale_staging).expect("stale staging directory should be created"); + let marker = stale_staging.join("preserve.txt"); + fs::write(&marker, "not owned by this invocation").expect("stale marker should be written"); + + let report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("unrelated stale staging should not block a build"); + + assert!(verify(&state, Some(&report.build_id), None).is_ok()); + assert_eq!( + fs::read_to_string(marker).expect("stale marker should remain"), + "not owned by this invocation" + ); + assert!(stale_staging.is_dir()); +} + +#[test] +fn manifest_validation_rejects_missing_version_unsupported_schema_and_traversal() { + for (case_id, mutation, expected) in [ + ("missing-version", "missing-version", "version manifest"), + ( + "unsupported-schema", + "unsupported-schema", + "unsupported ManifestVersion", + ), + ("unsafe-identifier", "unsafe-identifier", "unsafe"), + ("invalid-version", "invalid-version", "PackageVersion is invalid"), + ("duplicate-version", "duplicate-version", "duplicate"), + ("duplicate-locale", "duplicate-locale", "duplicate"), + ("duplicate-singleton", "duplicate-singleton", "duplicate"), + ("missing-installer", "missing-installer", "installer"), + ("empty-default-locale", "empty-default-locale", "DefaultLocale"), + ("duplicate-default-locale", "duplicate-default-locale", "duplicate"), + ("missing-singleton-locale", "missing-singleton-locale", "PackageLocale"), + ] { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + apply_mutation(&input, mutation); + let result = build(&BuildOptions { + input, + state: temp.path.join("state"), + }); + let error = result.expect_err(case_id); + assert!( + error + .to_string() + .to_ascii_lowercase() + .contains(&expected.to_ascii_lowercase()), + "{case_id}: unexpected diagnostic: {error}" + ); + } +} + +#[test] +fn manifest_validation_rejects_case_conflicting_package_ids() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + fs::write( + input.join("case-conflict.yaml"), + singleton_manifest("pinget.test.matrix", "2.0.0"), + ) + .expect("case-conflicting manifest should be written"); + let error = build(&BuildOptions { + input, + state: temp.path.join("state"), + }) + .expect_err("package identifiers that differ only by case must be rejected"); + + assert!( + error.to_string().to_ascii_lowercase().contains("cas"), + "unexpected case conflict diagnostic: {error}" + ); +} + +#[test] +fn overlapping_input_and_state_roots_are_rejected_before_state_creation() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = input.join("state"); + let error = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect_err("state must not overlap manifest inputs"); + + assert!(error.to_string().to_ascii_lowercase().contains("overlap")); + assert!(!state.exists()); +} + +#[cfg(windows)] +#[test] +fn manifest_scan_rejects_reparse_point_without_following_it() { + use std::os::windows::fs::symlink_dir; + + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let external = temp.path.join("external"); + fs::create_dir_all(&external).expect("external directory should be created"); + let link = input.join("linked"); + symlink_dir(&external, &link).expect("test environment must permit directory symlinks"); + let state = temp.path.join("state"); + + let error = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect_err("manifest roots must not follow reparse points"); + + assert!(error.to_string().to_ascii_lowercase().contains("reparse")); + assert!(!state.join("publisher.db").exists()); +} + +#[cfg(windows)] +#[test] +fn existing_writer_lock_prevents_build_mutation() { + use std::os::windows::fs::OpenOptionsExt; + + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + fs::create_dir_all(&state).expect("state root should be created"); + let held_lock = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .share_mode(0) + .open(state.join(".publisher.lock")) + .expect("exclusive writer lock should be held"); + + let error = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect_err("a second writer must not mutate locked state"); + + assert!(matches!( + error, + pinget_publisher::PublisherError::Io(_) | pinget_publisher::PublisherError::Invalid(_) + )); + assert!(state.join(".publisher.lock").is_file()); + assert!(!state.join("publisher.db").exists()); + drop(held_lock); +} + +#[test] +fn publish_unsigned_is_idempotent_and_excludes_staging_files_from_inventory() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let destination = temp.path.join("published"); + let build_report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("fixture should build before publication"); + + let first = publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.0", + &build_report.build_id, + )) + .expect("unsigned-development publication should succeed"); + assert!(first.valid); + assert_eq!(first.build_id, build_report.build_id); + assert!(!first.reused); + + let marker: Value = serde_json::from_slice( + &fs::read(destination.join(".pinget-publisher.json")).expect("ownership marker should exist"), + ) + .expect("ownership marker should be JSON"); + assert_eq!(marker["schemaVersion"].as_u64(), Some(1)); + assert!(marker.get("artifactFormatVersion").is_none()); + let artifacts = marker["artifacts"] + .as_object() + .expect("ownership marker should list published artifacts"); + assert!(artifacts.contains_key("source2.msix")); + assert!( + artifacts + .keys() + .all(|path| !path.contains(".source2.") && !path.contains(".publish-")), + "temporary staged package paths must not be recorded: {artifacts:?}" + ); + + let retry = publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.0", + &build_report.build_id, + )) + .expect("same version/build retry should be idempotent"); + assert!(retry.valid); + assert!(retry.reused); + assert_eq!(retry.artifact_count, artifacts.len()); + let verified = + verify_published_destination(&destination, None).expect("published destination should verify independently"); + assert!(verified.valid); + assert_eq!(verified.build_id.as_deref(), Some(build_report.build_id.as_str())); + assert_eq!(verified.artifact_count, artifacts.len()); +} + +#[test] +fn publish_rejects_downgrade_and_retains_immutable_artifacts_on_update() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let destination = temp.path.join("published"); + let first_build = build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("initial fixture should build"); + let first = publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.0", + &first_build.build_id, + )) + .expect("initial unsigned publication should succeed"); + let first_index = fs::read(destination.join("source2.msix")).expect("first index should exist"); + let snapshot: Value = serde_json::from_slice( + &fs::read(first_build.snapshot_directory.join("snapshot.json")).expect("snapshot metadata should exist"), + ) + .expect("snapshot metadata should be JSON"); + let old_artifacts = snapshot["artifacts"] + .as_object() + .expect("snapshot should list immutable feed artifacts") + .keys() + .cloned() + .collect::>(); + + let changed = input.join("Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + let mut contents = fs::read_to_string(&changed).expect("manifest should be readable"); + contents.push_str("# publication content update\n"); + fs::write(&changed, contents).expect("manifest content should change"); + let second_build = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("updated input should build"); + assert_ne!(first_build.build_id, second_build.build_id); + + let downgrade = publish(&unsigned_publish_options( + &state, + &destination, + "0.9.9.9", + &second_build.build_id, + )) + .expect_err("package version must not move backwards"); + assert!( + downgrade.to_string().contains("must be greater"), + "unexpected downgrade diagnostic: {downgrade}" + ); + assert_eq!( + first_index, + fs::read(destination.join("source2.msix")).expect("old MSIX should remain after rejection") + ); + + let updated = publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.1", + &second_build.build_id, + )) + .expect("monotonic package version should publish"); + assert!(updated.valid); + assert_eq!(updated.build_id, second_build.build_id); + assert_ne!( + first_index, + fs::read(destination.join("source2.msix")).expect("updated MSIX should exist") + ); + for relative in old_artifacts { + assert!( + destination.join(&relative).is_file(), + "previous immutable artifact was not retained: {relative}" + ); + } + assert!( + verify_published_destination(&destination, None) + .expect("updated destination should verify") + .valid + ); + assert!(first.valid); +} + +#[test] +fn publish_rejects_unowned_destination_without_overwriting_unrelated_file() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let destination = temp.path.join("published"); + let build_report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("fixture should build before publication"); + fs::create_dir_all(&destination).expect("unowned destination should be created"); + let unrelated = destination.join("keep-me.txt"); + let original = b"unrelated destination content"; + fs::write(&unrelated, original).expect("unrelated file should be written"); + + let error = publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.0", + &build_report.build_id, + )) + .expect_err("publisher must not adopt a non-empty unowned directory"); + + assert!( + error.to_string().to_ascii_lowercase().contains("unowned"), + "unexpected unowned-destination diagnostic: {error}" + ); + assert_eq!(fs::read(&unrelated).expect("unrelated file should remain"), original); + assert!(!destination.join(".pinget-publisher.json").exists()); + assert!(!destination.join("source2.msix").exists()); + assert!( + status(&state) + .expect("state should remain readable") + .last_published_build_id + .is_none() + ); +} + +#[test] +fn publish_rejects_identity_change_and_untracked_artifact_without_mutation() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let destination = temp.path.join("published"); + let first_build = build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("fixture should build before publication"); + let options = unsigned_publish_options(&state, &destination, "1.0.0.0", &first_build.build_id); + publish(&options).expect("initial unsigned publication should succeed"); + let original_index = fs::read(destination.join("source2.msix")).expect("index should exist"); + let original_marker = fs::read(destination.join(".pinget-publisher.json")).expect("marker should exist"); + + let mut changed_identity = unsigned_publish_options(&state, &destination, "1.0.0.0", &first_build.build_id); + changed_identity.name = "Pinget.Publisher.Other".to_owned(); + let identity_error = publish(&changed_identity).expect_err("owned package identity must remain stable"); + assert!( + identity_error.to_string().contains("different package identity"), + "unexpected identity diagnostic: {identity_error}" + ); + assert_eq!( + fs::read(destination.join("source2.msix")).expect("MSIX should remain after identity rejection"), + original_index + ); + assert_eq!( + fs::read(destination.join(".pinget-publisher.json")) + .expect("ownership marker should remain after identity rejection"), + original_marker + ); + + let unrelated = destination.join("keep-me.txt"); + let unrelated_bytes = b"untracked owned-destination content"; + fs::write(&unrelated, unrelated_bytes).expect("untracked file should be written"); + let verify_error = verify_published_destination(&destination, None) + .expect_err("destination verification must reject untracked artifacts"); + assert!( + verify_error.to_string().to_ascii_lowercase().contains("untracked"), + "unexpected verification diagnostic: {verify_error}" + ); + + let changed_manifest = input.join("Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + let mut contents = fs::read_to_string(&changed_manifest).expect("manifest should be readable"); + contents.push_str("# attempted update with untracked artifact\n"); + fs::write(&changed_manifest, contents).expect("manifest content should change"); + let second_build = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("updated input should build"); + let publish_error = publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.1", + &second_build.build_id, + )) + .expect_err("publish must reject untracked destination files"); + assert!( + publish_error.to_string().to_ascii_lowercase().contains("untracked"), + "unexpected publish diagnostic: {publish_error}" + ); + assert_eq!( + fs::read(&unrelated).expect("untracked file should remain"), + unrelated_bytes + ); + assert_eq!( + fs::read(destination.join("source2.msix")).expect("MSIX should remain after rejection"), + original_index + ); + assert_eq!( + fs::read(destination.join(".pinget-publisher.json")).expect("ownership marker should remain after rejection"), + original_marker + ); +} + +#[test] +fn publish_failed_packaging_tool_does_not_publish_or_record_build() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let destination = temp.path.join("published"); + let build_report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("fixture should build before packaging"); + let where_tool = PathBuf::from(std::env::var_os("WINDIR").expect("Windows system root")) + .join("System32") + .join("where.exe"); + assert!(where_tool.is_file(), "where.exe should be available on Windows"); + let mut options = unsigned_publish_options(&state, &destination, "1.0.0.0", &build_report.build_id); + options.makeappx = Some(where_tool); + + let error = publish(&options).expect_err("the controlled failing pack tool must fail"); + + assert!( + error.to_string().contains("failed"), + "unexpected packaging diagnostic: {error}" + ); + assert!(!destination.join("source2.msix").exists()); + assert!(!destination.join(".pinget-publisher.json").exists()); + assert!( + status(&state) + .expect("state should remain readable") + .last_published_build_id + .is_none() + ); + + options.makeappx = doctor().makeappx_path; + let retry = publish(&options).expect("retry with the discovered SDK packager should succeed"); + assert!(retry.valid); + assert_eq!(retry.build_id, build_report.build_id); + assert!( + verify_published_destination(&destination, None) + .expect("retried destination should verify") + .valid + ); + assert_eq!( + status(&state) + .expect("state should record the successful retry") + .last_published_build_id, + Some(build_report.build_id) + ); +} + +#[cfg(windows)] +#[test] +fn publish_semantic_metadata_promotion_failure_preserves_old_feed_then_retry_succeeds() { + use std::os::windows::fs::OpenOptionsExt; + + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let destination = temp.path.join("published"); + let first_build = build(&BuildOptions { + input: input.clone(), + state: state.clone(), + }) + .expect("initial fixture should build"); + publish(&unsigned_publish_options( + &state, + &destination, + "1.0.0.0", + &first_build.build_id, + )) + .expect("initial publication should succeed"); + let original_index = fs::read(destination.join("source2.msix")).expect("index should exist"); + let original_marker = fs::read(destination.join(".pinget-publisher.json")).expect("marker should exist"); + + let changed_manifest = input.join("Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + let contents = fs::read_to_string(&changed_manifest).expect("manifest should be readable"); + let updated_contents = contents.replace("PackageName: Matrix Package", "PackageName: Matrix Package Updated"); + assert_ne!( + contents, updated_contents, + "the input mutation must change real metadata" + ); + fs::write(&changed_manifest, updated_contents).expect("manifest metadata should change"); + let second_build = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("updated fixture should build"); + let next = unsigned_publish_options(&state, &destination, "1.0.0.1", &second_build.build_id); + + let locked_index = fs::OpenOptions::new() + .read(true) + .share_mode(1) + .open(destination.join("source2.msix")) + .expect("read-only handle should prevent atomic replacement"); + let error = publish(&next).expect_err("root index promotion must fail while index is open"); + drop(locked_index); + + assert!(error.to_string().to_ascii_lowercase().contains("access")); + assert_eq!( + fs::read(destination.join("source2.msix")).expect("MSIX should remain after rollback"), + original_index + ); + assert_eq!( + fs::read(destination.join(".pinget-publisher.json")).expect("ownership marker should remain after rollback"), + original_marker + ); + assert!( + verify_published_destination(&destination, None) + .expect("rolled-back destination should remain verifiable") + .valid + ); + assert_eq!( + status(&state) + .expect("state should remain readable after rollback") + .last_published_build_id + .as_deref(), + Some(first_build.build_id.as_str()), + "failed promotion must not record the new build as published" + ); + + let retry = publish(&next).expect("retry after releasing the blocking handle should succeed"); + assert!(retry.valid); + assert_eq!(retry.build_id, second_build.build_id); + assert_ne!( + original_index, + fs::read(destination.join("source2.msix")).expect("retry should promote the new index") + ); + assert_ne!( + original_marker, + fs::read(destination.join(".pinget-publisher.json")).expect("retry should update ownership metadata") + ); + assert!( + verify_published_destination(&destination, None) + .expect("retried destination should verify") + .valid + ); + assert_eq!( + status(&state) + .expect("state should reflect successful retry") + .last_published_build_id, + Some(second_build.build_id) + ); +} + +#[test] +fn publish_destination_lock_serializes_different_state_roots_and_allows_retry() { + use std::os::windows::fs::OpenOptionsExt; + + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let first_state = temp.path.join("first-state"); + let second_state = temp.path.join("second-state"); + let destination = temp.path.join("published"); + let first_build = build(&BuildOptions { + input: input.clone(), + state: first_state.clone(), + }) + .expect("first state should build"); + let second_build = build(&BuildOptions { + input, + state: second_state.clone(), + }) + .expect("second state should build"); + assert_eq!(first_build.build_id, second_build.build_id); + publish(&unsigned_publish_options( + &first_state, + &destination, + "1.0.0.0", + &first_build.build_id, + )) + .expect("first state should publish"); + let old_index = fs::read(destination.join("source2.msix")).expect("old index should exist"); + let old_marker = fs::read(destination.join(".pinget-publisher.json")).expect("old marker should exist"); + let destination_name = destination + .file_name() + .expect("destination should have a final path component") + .to_string_lossy(); + let lock_path = destination.with_file_name(format!("{destination_name}.pinget-publisher.lock")); + + let held_lock = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .share_mode(0) + .open(lock_path) + .expect("destination sibling lock should be acquired"); + let error = publish(&unsigned_publish_options( + &second_state, + &destination, + "1.0.0.1", + &second_build.build_id, + )) + .expect_err("a second state root must respect the destination lock"); + assert!(error.to_string().to_ascii_lowercase().contains("lock")); + assert_eq!( + fs::read(destination.join("source2.msix")).expect("old index should remain"), + old_index + ); + assert_eq!( + fs::read(destination.join(".pinget-publisher.json")).expect("old marker should remain"), + old_marker + ); + assert!( + verify_published_destination(&destination, None) + .expect("destination should remain valid during lock rejection") + .valid + ); + drop(held_lock); + + let retry = publish(&unsigned_publish_options( + &second_state, + &destination, + "1.0.0.1", + &second_build.build_id, + )) + .expect("publication should succeed after the destination lock is released"); + assert!(retry.valid); + assert_eq!(retry.build_id, second_build.build_id); + assert!( + verify_published_destination(&destination, None) + .expect("retried destination should verify") + .valid + ); + assert_eq!( + status(&second_state) + .expect("second state should record publication") + .last_published_build_id, + Some(second_build.build_id) + ); +} + +#[test] +fn verify_rejects_corrupted_snapshot_sidecar_and_index() { + for (component, location) in [ + ("snapshot", ""), + ("sidecar", "feed/packages"), + ("manifest", "feed/manifests"), + ("index", "Public"), + ] { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + let report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("fixture should build before corruption"); + let snapshot = state.join("builds").join(&report.build_id); + let file = match component { + "snapshot" => snapshot.join("snapshot.json"), + "index" => snapshot.join("Public").join("index.db"), + _ => find_first_file(&snapshot.join(location)), + }; + let corrupted = if component == "snapshot" { + b"{not-json".as_slice() + } else { + b"tampered bytes".as_slice() + }; + fs::write(&file, corrupted).expect("snapshot file should be mutated"); + + assert!( + verify(&state, Some(&report.build_id), None).is_err(), + "corrupted {component} should fail verification" + ); + } +} + +#[test] +fn status_rejects_corrupted_state_database_without_reinitializing_it() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let state = temp.path.join("state"); + build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("fixture should build before state corruption"); + let database = state.join("publisher.db"); + let corrupted = b"not a sqlite database"; + fs::write(&database, corrupted).expect("state database should be corrupted"); + + assert!(status(&state).is_err()); + assert_eq!( + fs::read(&database).expect("corrupted database should remain in place"), + corrupted + ); +} + +fn apply_mutation(input: &Path, mutation: &str) { + let version = input.join("Pinget.Test.Matrix_1.10.0.version.yaml"); + match mutation { + "missing-version" => fs::remove_file(version).expect("version manifest should be removed"), + "unsupported-schema" => { + let content = fs::read_to_string(&version).expect("version manifest should be readable"); + fs::write( + &version, + content.replace("ManifestVersion: 1.9.0", "ManifestVersion: 9.9.9"), + ) + .expect("unsupported schema should be written"); + } + "unsafe-identifier" => { + let content = fs::read_to_string(&version).expect("version manifest should be readable"); + fs::write(&version, content.replace("Pinget.Test.Matrix", "../escape")) + .expect("traversal identifier should be written"); + } + "invalid-version" => { + let content = fs::read_to_string(&version).expect("version manifest should be readable"); + fs::write( + &version, + content.replace("PackageVersion: 1.10.0", "PackageVersion: ../escape"), + ) + .expect("invalid package version should be written"); + } + "duplicate-version" => { + fs::copy(&version, input.join("duplicate-version.yaml")) + .expect("duplicate version manifest should be copied"); + } + "duplicate-locale" => { + fs::copy( + input.join("Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml"), + input.join("duplicate-fr.yaml"), + ) + .expect("duplicate locale manifest should be copied"); + } + "duplicate-singleton" => { + fs::copy( + input.join("Pinget.Test.Matrix_1.9.0.yaml"), + input.join("duplicate-singleton.yaml"), + ) + .expect("duplicate singleton manifest should be copied"); + } + "missing-installer" => { + fs::remove_file(input.join("Pinget.Test.Matrix_1.10.0.installer.yaml")) + .expect("installer manifest should be removed"); + } + "empty-default-locale" => { + let content = fs::read_to_string(&version).expect("version manifest should be readable"); + fs::write(&version, content.replace("DefaultLocale: en-US", "DefaultLocale: ''")) + .expect("empty default locale should be written"); + } + "duplicate-default-locale" => { + let default_locale = input.join("Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); + let mut content = fs::read_to_string(default_locale).expect("default locale should be readable"); + content = content.replace("PackageLocale: en-US", "PackageLocale: en-us"); + fs::write(input.join("duplicate-default-locale.yaml"), content) + .expect("duplicate default locale should be written"); + } + "missing-singleton-locale" => { + let content = + singleton_manifest("Pinget.Test.MissingLocale", "1.0.0").replace("PackageLocale: en-US\n", ""); + fs::write(input.join("missing-locale.yaml"), content) + .expect("singleton without package locale should be written"); + } + _ => unreachable!("only declared test mutations are used"), + } +} + +fn unsigned_publish_options(state: &Path, destination: &Path, version: &str, build_id: &str) -> PublishOptions { + let makeappx = makeappx_tool(); + PublishOptions { + state: state.to_path_buf(), + build_id: Some(build_id.to_owned()), + destination: destination.to_path_buf(), + name: "Pinget.Publisher.Conformance".to_owned(), + publisher: "CN=Pinget Publisher Conformance Test".to_owned(), + version: version.to_owned(), + signing: Signing::UnsignedDevelopment, + makeappx: Some(makeappx), + signtool: None, + timestamp_url: None, + } +} + +fn makeappx_tool() -> PathBuf { + if let Some(path) = doctor().makeappx_path.filter(|path| path.is_file()) { + return path; + } + + let program_files = std::env::var_os("ProgramFiles(x86)") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(r"C:\Program Files (x86)")); + let sdk_root = program_files.join("Windows Kits").join("10").join("bin"); + let mut sdk_versions = fs::read_dir(&sdk_root) + .unwrap_or_else(|error| panic!("Windows SDK bin directory is unavailable: {error}")) + .filter_map(Result::ok) + .map(|entry| entry.path()) + .collect::>(); + sdk_versions.sort(); + sdk_versions.reverse(); + sdk_versions + .into_iter() + .map(|version| version.join("x64").join("makeappx.exe")) + .find(|path| path.is_file()) + .expect("Windows SDK MakeAppx must be available to test publication") +} + +fn singleton_manifest(id: &str, version: &str) -> String { + format!( + "PackageIdentifier: {id}\n\ + PackageVersion: {version}\n\ + PackageLocale: en-US\n\ + Publisher: Pinget Test Fixtures\n\ + PackageName: {id}\n\ + License: MIT\n\ + ShortDescription: temporary fixture\n\ + Installers:\n\ + - Architecture: x64\n\ + InstallerType: exe\n\ + InstallerUrl: https://example.invalid/fixture.exe\n\ + InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000\n\ + ManifestType: singleton\n\ + ManifestVersion: 1.9.0\n" + ) +} + +fn write_locale_order_manifests(input: &Path, id: &str) { + for (suffix, locale, publisher, name, kind) in [ + ( + "locale.en-US", + "en-US", + "Pinget Test Fixtures", + "Locale Order English", + "defaultLocale", + ), + ( + "locale.de-DE", + "de-DE", + "Pinget Test Fixtures", + "Locale Reihenfolge", + "locale", + ), + ] { + fs::write( + input.join(format!("{id}_1.0.0.{suffix}.yaml")), + format!( + "PackageIdentifier: {id}\n\ + PackageVersion: 1.0.0\n\ + PackageLocale: {locale}\n\ + Publisher: {publisher}\n\ + PackageName: {name}\n\ + License: MIT\n\ + ShortDescription: locale-order fixture\n\ + ManifestType: {kind}\n\ + ManifestVersion: 1.9.0\n" + ), + ) + .expect("locale manifest should be written"); + } + fs::write( + input.join(format!("{id}_1.0.0.version.yaml")), + format!( + "PackageIdentifier: {id}\n\ + PackageVersion: 1.0.0\n\ + DefaultLocale: en-US\n\ + ManifestType: version\n\ + ManifestVersion: 1.9.0\n" + ), + ) + .expect("version manifest should be written"); + fs::write( + input.join(format!("{id}_1.0.0.installer.yaml")), + format!( + "PackageIdentifier: {id}\n\ + PackageVersion: 1.0.0\n\ + Installers:\n\ + - Architecture: x64\n\ + InstallerType: exe\n\ + InstallerUrl: https://example.invalid/locale-order.exe\n\ + InstallerSha256: 1111111111111111111111111111111111111111111111111111111111111111\n\ + ManifestType: installer\n\ + ManifestVersion: 1.9.0\n" + ), + ) + .expect("installer manifest should be written"); +} + +fn copy_fixture(name: &str, destination: &Path) -> PathBuf { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/publisher/fixtures") + .join(name); + copy_directory(&source, destination); + destination.to_path_buf() +} + +fn copy_directory(source: &Path, destination: &Path) { + fs::create_dir_all(destination).expect("fixture destination should be created"); + for entry in fs::read_dir(source).expect("fixture directory should be readable") { + let entry = entry.expect("fixture entry should be readable"); + let path = entry.path(); + let target = destination.join(entry.file_name()); + if path.is_dir() { + copy_directory(&path, &target); + } else { + fs::copy(path, target).expect("fixture file should be copied"); + } + } +} + +fn find_first_file(directory: &Path) -> PathBuf { + let mut pending = vec![directory.to_path_buf()]; + while let Some(current) = pending.pop() { + for entry in fs::read_dir(current).expect("snapshot directory should be readable") { + let path = entry.expect("snapshot entry should be readable").path(); + if path.is_dir() { + pending.push(path); + } else { + return path; + } + } + } + panic!("expected at least one snapshot file"); +} + +fn count_files_with_extension(directory: &Path, extension: &str) -> usize { + let mut pending = vec![directory.to_path_buf()]; + let mut count = 0; + while let Some(current) = pending.pop() { + for entry in fs::read_dir(current).expect("snapshot directory should be readable") { + let path = entry.expect("snapshot entry should be readable").path(); + if path.is_dir() { + pending.push(path); + } else if path.extension().and_then(|value| value.to_str()) == Some(extension) { + count += 1; + } + } + } + count +} + +fn sha256_file(path: &Path) -> String { + use sha2::{Digest, Sha256}; + encode_hex(&Sha256::digest(fs::read(path).expect("sidecar should be readable"))) +} + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + encode_hex(&Sha256::digest(bytes)) +} + +fn read_correlations(database: &Connection, table: &str, column: &str) -> Vec { + let query = format!( + "SELECT {column} FROM {table} AS valuesTable \ + JOIN packages AS package ON package.rowid=valuesTable.package \ + WHERE package.id=?1 ORDER BY {column}" + ); + let mut statement = database.prepare(&query).expect("correlation query prepares"); + statement + .query_map(["Pinget.Test.Matrix"], |row| row.get(0)) + .expect("correlation query executes") + .collect::, _>>() + .expect("correlation rows should decode") +} + +fn read_mapped_values( + database: &Connection, + value_table: &str, + value_column: &str, + map_table: &str, + map_column: &str, +) -> Vec { + let query = format!( + "SELECT value.{value_column} FROM {value_table} AS value \ + JOIN {map_table} AS value_map ON value_map.{map_column}=value.rowid \ + JOIN packages AS package ON package.rowid=value_map.package \ + WHERE package.id=?1 ORDER BY value.{value_column}" + ); + let mut statement = database.prepare(&query).expect("mapping query prepares"); + statement + .query_map(["Pinget.Test.Matrix"], |row| row.get(0)) + .expect("mapping query executes") + .collect::, _>>() + .expect("mapping values should decode") +} + +fn encode_hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(char::from(DIGITS[usize::from(byte >> 4)])); + output.push(char::from(DIGITS[usize::from(byte & 0x0f)])); + } + output +} + +struct TempDirectory { + path: PathBuf, +} + +impl TempDirectory { + fn new() -> Self { + let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir() + .join("pinget-publisher-rust-tests") + .join(format!("{}-{sequence}", std::process::id())); + fs::create_dir_all(&path).expect("test scratch directory should be created"); + Self { path } + } +} + +impl Drop for TempDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} diff --git a/rust/release-plz.toml b/rust/release-plz.toml index 59b7a9b..1ac63fb 100644 --- a/rust/release-plz.toml +++ b/rust/release-plz.toml @@ -6,12 +6,16 @@ semver_check = true pr_branch_prefix = "release-plz/" pr_name = "chore(release): prepare for publishing" -# For now, only publish pinget-core to crates.io. +# Publish the reusable core and publisher crates to crates.io. [[package]] name = "pinget-cli" publish = false release = false +[[package]] +name = "pinget-publisher" +publish = true + [[package]] name = "pinget-com" publish = false diff --git a/scripts/Set-PingetVersion.ps1 b/scripts/Set-PingetVersion.ps1 index d74a5ae..b992ee5 100644 --- a/scripts/Set-PingetVersion.ps1 +++ b/scripts/Set-PingetVersion.ps1 @@ -60,6 +60,21 @@ Set-VersionInFile ` -Pattern '(pinget-core\s*=\s*\{\s*version\s*=\s*")[^"]+(")' ` -Replacement { param($match) "$($match.Groups[1].Value)$Version$($match.Groups[2].Value)" } +Set-VersionInFile ` + -RelativePath 'rust\crates\pinget-publisher\Cargo.toml' ` + -Pattern '^(version\s*=\s*")[^"]+(")' ` + -Replacement { param($match) "$($match.Groups[1].Value)$Version$($match.Groups[2].Value)" } + +Set-VersionInFile ` + -RelativePath 'rust\crates\pinget-publisher\Cargo.toml' ` + -Pattern '(pinget-core\s*=\s*\{\s*version\s*=\s*")[^"]+(")' ` + -Replacement { param($match) "$($match.Groups[1].Value)$Version$($match.Groups[2].Value)" } + +Set-VersionInFile ` + -RelativePath 'rust\crates\pinget-cli\Cargo.toml' ` + -Pattern '(pinget-publisher\s*=\s*\{\s*version\s*=\s*")[^"]+(")' ` + -Replacement { param($match) "$($match.Groups[1].Value)$Version$($match.Groups[2].Value)" } + Set-VersionInFile ` -RelativePath 'rust\crates\pinget-com\Cargo.toml' ` -Pattern '^(version\s*=\s*")[^"]+(")' ` @@ -67,7 +82,7 @@ Set-VersionInFile ` $cargoLockPath = Join-Path $repoRoot 'rust\Cargo.lock' if (Test-Path -Path $cargoLockPath) { - foreach ($crateName in @('pinget-core', 'pinget-cli', 'pinget-com')) { + foreach ($crateName in @('pinget-core', 'pinget-cli', 'pinget-com', 'pinget-publisher')) { Set-VersionInFile ` -RelativePath 'rust\Cargo.lock' ` -Pattern "(\[\[package\]\]\r?\nname = `"$crateName`"\r?\nversion = `")[^`"]+(`")" ` diff --git a/tests/publisher/bootstrap-winget-sandbox.ps1 b/tests/publisher/bootstrap-winget-sandbox.ps1 new file mode 100644 index 0000000..e172f39 --- /dev/null +++ b/tests/publisher/bootstrap-winget-sandbox.ps1 @@ -0,0 +1,116 @@ +param( + [string] $WorkRoot = 'C:\WinApp\work\PublisherContract' +) + +$ErrorActionPreference = 'Stop' +$bundlePath = Join-Path $WorkRoot 'bootstrap\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle' +$certificatePath = Join-Path $WorkRoot 'publisher-contract.cer' + +$signerCertificate = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new( + $certificatePath +) +$trustedPeople = [System.Security.Cryptography.X509Certificates.X509Store]::new( + [System.Security.Cryptography.X509Certificates.StoreName]::TrustedPeople, + [System.Security.Cryptography.X509Certificates.StoreLocation]::CurrentUser +) +try { + $trustedPeople.Open( + [System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite + ) + $alreadyTrusted = $trustedPeople.Certificates | + Where-Object { $_.Thumbprint -eq $signerCertificate.Thumbprint } + if ($null -eq $alreadyTrusted) { + $trustedPeople.Add($signerCertificate) + } +} +finally { + $trustedPeople.Close() + $signerCertificate.Dispose() +} +Write-Output 'Trusted ephemeral fixture signer in the Sandbox user Trusted People store.' + +$rsa = [System.Security.Cryptography.RSA]::Create(2048) +$serverRequest = [System.Security.Cryptography.X509Certificates.CertificateRequest]::new( + 'CN=127.0.0.1', + $rsa, + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1 +) +$serverRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509BasicConstraintsExtension]::new( + $false, $false, 0, $true + ) +) +$serverRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509KeyUsageExtension]::new( + [System.Security.Cryptography.X509Certificates.X509KeyUsageFlags]::DigitalSignature, + $true + ) +) +$serverEku = [System.Security.Cryptography.OidCollection]::new() +[void]$serverEku.Add( + [System.Security.Cryptography.Oid]::new('1.3.6.1.5.5.7.3.1') +) +$serverRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509EnhancedKeyUsageExtension]::new( + $serverEku, $false + ) +) +$san = [System.Security.Cryptography.X509Certificates.SubjectAlternativeNameBuilder]::new() +$san.AddIpAddress([System.Net.IPAddress]::Parse('127.0.0.1')) +$serverRequest.CertificateExtensions.Add($san.Build()) +$tlsCertificate = $serverRequest.CreateSelfSigned( + [DateTimeOffset]::UtcNow.AddMinutes(-5), + [DateTimeOffset]::UtcNow.AddDays(2) +) +$tlsPath = Join-Path $WorkRoot 'tls-server.pfx' +[System.IO.File]::WriteAllBytes( + $tlsPath, + $tlsCertificate.Export( + [System.Security.Cryptography.X509Certificates.X509ContentType]::Pfx + ) +) +[System.IO.File]::WriteAllBytes( + (Join-Path $WorkRoot 'tls-server.cer'), + $tlsCertificate.Export( + [System.Security.Cryptography.X509Certificates.X509ContentType]::Cert + ) +) +$tlsTrustedPeople = [System.Security.Cryptography.X509Certificates.X509Store]::new( + [System.Security.Cryptography.X509Certificates.StoreName]::TrustedPeople, + [System.Security.Cryptography.X509Certificates.StoreLocation]::CurrentUser +) +try { + $tlsTrustedPeople.Open( + [System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite + ) + $tlsTrustedPeople.Add($tlsCertificate) +} +finally { + $tlsTrustedPeople.Close() + $tlsCertificate.Dispose() + $rsa.Dispose() +} +Write-Output 'Created and trusted the loopback TLS certificate in Sandbox only.' + +$dependencyDirectory = Join-Path $WorkRoot 'bootstrap\selected-x64' +if (-not (Test-Path -LiteralPath $dependencyDirectory)) { + $dependencyDirectory = Join-Path $WorkRoot 'bootstrap' +} +$dependencies = Get-ChildItem -LiteralPath $dependencyDirectory ` + -Filter '*.appx' | ForEach-Object FullName +$appInstaller = Get-AppxPackage -Name Microsoft.DesktopAppInstaller +if ($null -eq $appInstaller -or $appInstaller.Version.ToString() -ne '1.29.380.0') { + Add-AppxPackage -Path $bundlePath -DependencyPath $dependencies + $appInstaller = Get-AppxPackage -Name Microsoft.DesktopAppInstaller +} +if ($null -eq $appInstaller -or $appInstaller.Version.ToString() -ne '1.29.380.0') { + $actualVersion = if ($null -eq $appInstaller) { + '' + } + else { + $appInstaller.Version.ToString() + } + throw "Expected official WinGet 1.29.380.0, found $actualVersion" +} +Write-Output "Installed official WinGet $($appInstaller.Version)" diff --git a/tests/publisher/build_v2_fixture.py b/tests/publisher/build_v2_fixture.py new file mode 100644 index 0000000..40f0595 --- /dev/null +++ b/tests/publisher/build_v2_fixture.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +"""Build and self-check a locally authored WinGet schema 2.0 source fixture.""" + +from __future__ import annotations + +import argparse +import ctypes +import hashlib +import sqlite3 +import subprocess +import sys +import unittest +import uuid +import zipfile +from pathlib import Path +from xml.etree import ElementTree + + +ROOT = Path(__file__).resolve().parent +FIXTURES = ROOT / "fixtures" +SCHEMA = ROOT / "index-schema.sql" +PAYLOAD = b"Harmless Pinget contract fixture. Download only; never execute.\n" +MANIFEST_RELATIVE = "manifests/Pinget.Contract.Singleton.yaml" +SIDECAR_RELATIVE = ( + "packages/Pinget.Contract.Singleton/{hash_prefix}/versionData.mszyml" +) +HASH_RE = set("0123456789abcdef") +ARTIFACTS: Path | None = None + + +def sha256(data: bytes) -> bytes: + return hashlib.sha256(data).digest() + + +def compress_mszip(data: bytes) -> bytes: + """Use the same native Windows Compression API as WinGet's producer.""" + api = ctypes.WinDLL("cabinet.dll", use_last_error=True) + handle = ctypes.c_void_p() + api.CreateCompressor.argtypes = [ + ctypes.c_uint32, + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ] + api.CreateCompressor.restype = ctypes.c_int + api.Compress.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), + ] + api.Compress.restype = ctypes.c_int + api.CloseCompressor.argtypes = [ctypes.c_void_p] + api.CloseCompressor.restype = ctypes.c_int + + if not api.CreateCompressor(2, None, ctypes.byref(handle)): + raise ctypes.WinError(ctypes.get_last_error()) + try: + source = ctypes.create_string_buffer(data) + required = ctypes.c_size_t() + api.Compress( + handle, source, len(data), None, 0, ctypes.byref(required) + ) + if ctypes.get_last_error() != 122: + raise ctypes.WinError(ctypes.get_last_error()) + output = ctypes.create_string_buffer(required.value) + actual = ctypes.c_size_t() + if not api.Compress( + handle, + source, + len(data), + output, + required.value, + ctypes.byref(actual), + ): + raise ctypes.WinError(ctypes.get_last_error()) + return output.raw[: actual.value] + finally: + api.CloseCompressor(handle) + + +def decompress_mszip(data: bytes) -> bytes: + """Independently verify the stored stream using the native decompressor.""" + api = ctypes.WinDLL("cabinet.dll", use_last_error=True) + handle = ctypes.c_void_p() + api.CreateDecompressor.argtypes = [ + ctypes.c_uint32, + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ] + api.CreateDecompressor.restype = ctypes.c_int + api.Decompress.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), + ] + api.Decompress.restype = ctypes.c_int + api.CloseDecompressor.argtypes = [ctypes.c_void_p] + api.CloseDecompressor.restype = ctypes.c_int + + if not api.CreateDecompressor(2, None, ctypes.byref(handle)): + raise ctypes.WinError(ctypes.get_last_error()) + try: + source = ctypes.create_string_buffer(data) + required = ctypes.c_size_t() + api.Decompress( + handle, source, len(data), None, 0, ctypes.byref(required) + ) + if ctypes.get_last_error() != 122: + raise ctypes.WinError(ctypes.get_last_error()) + output = ctypes.create_string_buffer(required.value) + actual = ctypes.c_size_t() + if not api.Decompress( + handle, + source, + len(data), + output, + required.value, + ctypes.byref(actual), + ): + raise ctypes.WinError(ctypes.get_last_error()) + return output.raw[: actual.value] + finally: + api.CloseDecompressor(handle) + + +def create_database(path: Path, compressed_sidecar: bytes) -> None: + connection = sqlite3.connect(path) + try: + connection.executescript(SCHEMA.read_text(encoding="utf-8")) + with connection: + connection.executemany( + "INSERT INTO metadata(name, value) VALUES(?, ?)", + [ + ("majorVersion", "2"), + ("minorVersion", "0"), + ("databaseIdentifier", "{" + str(uuid.uuid4()).upper() + "}"), + ("lastwritetime", "1793380000"), + ], + ) + connection.execute( + """INSERT INTO packages( + rowid, id, name, moniker, latest_version, + arp_min_version, arp_max_version, hash + ) VALUES(1, ?, ?, ?, ?, NULL, NULL, ?)""", + ( + "Pinget.Contract.Singleton", + "Pinget Contract Fixture", + "pinget-contract", + "1.0.0", + sqlite3.Binary(sha256(compressed_sidecar)), + ), + ) + connection.execute( + "INSERT INTO tags2(rowid, tag) VALUES(1, 'publisher-contract')" + ) + connection.execute( + "INSERT INTO tags2_map(tag, package) VALUES(1, 1)" + ) + finally: + connection.close() + + +def build(output: Path, makeappx: Path, publisher: str, installer_url: str) -> None: + global ARTIFACTS + ARTIFACTS = output.resolve() + if ARTIFACTS.exists(): + raise FileExistsError( + f"Refusing to overwrite fixture output directory: {ARTIFACTS}" + ) + feed = ARTIFACTS / "feed" + stage = ARTIFACTS / "package-staging" + (feed / "manifests").mkdir(parents=True) + (feed / "packages").mkdir() + (stage / "Public").mkdir(parents=True) + (stage / "Assets").mkdir() + + payload_path = feed / "HarmlessPayload.txt" + payload_path.write_bytes(PAYLOAD) + manifest_template = (FIXTURES / "singleton.yaml.in").read_text( + encoding="utf-8" + ) + manifest = manifest_template.replace("{{INSTALLER_URL}}", installer_url) + manifest = manifest.replace( + "{{INSTALLER_SHA256}}", hashlib.sha256(PAYLOAD).hexdigest() + ) + manifest_bytes = manifest.encode("utf-8") + (feed / MANIFEST_RELATIVE).write_bytes(manifest_bytes) + + version_data = ( + 'sV: "1.0"\n' + "vD:\n" + ' - v: "1.0.0"\n' + f" rP: {MANIFEST_RELATIVE}\n" + f" s256H: {hashlib.sha256(manifest_bytes).hexdigest()}\n" + ).encode("utf-8") + compressed_sidecar = compress_mszip(version_data) + sidecar_hash = hashlib.sha256(compressed_sidecar).hexdigest() + sidecar_relative = SIDECAR_RELATIVE.format(hash_prefix=sidecar_hash[:8]) + sidecar_path = feed / Path(sidecar_relative) + sidecar_path.parent.mkdir(parents=True) + sidecar_path.write_bytes(compressed_sidecar) + (ARTIFACTS / "versionData.yml").write_bytes(version_data) + + create_database(stage / "Public" / "index.db", compressed_sidecar) + + appx_template = (FIXTURES / "AppxManifest.xml.in").read_text( + encoding="utf-8" + ) + appx_manifest = appx_template.replace("{{PUBLISHER}}", publisher) + (stage / "AppxManifest.xml").write_text( + appx_manifest, encoding="utf-8", newline="\n" + ) + # Locally-authored, one-pixel transparent PNG used only as required package assets. + logo = bytes.fromhex( + "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489" + "0000000b49444154789c636000020000050001a5f645400000000049454e44ae426082" + ) + for name in ("StoreLogo.png", "Logo150.png", "Logo44.png"): + (stage / "Assets" / name).write_bytes(logo) + + package_path = feed / "source2.msix" + subprocess.run( + [str(makeappx), "pack", "/d", str(stage), "/p", str(package_path), "/o"], + check=True, + ) + (ARTIFACTS / "fixture-metadata.txt").write_text( + f"Package=source2.msix\n" + f"Index=Public/index.db\n" + f"Manifest={MANIFEST_RELATIVE}\n" + f"Sidecar={sidecar_relative}\n" + f"SidecarSHA256={sidecar_hash}\n" + f"InstallerURL={installer_url}\n" + f"InstallerSHA256={hashlib.sha256(PAYLOAD).hexdigest()}\n" + f"Publisher={publisher}\n", + encoding="utf-8", + ) + + +class V2FixtureTests(unittest.TestCase): + def setUp(self) -> None: + if ARTIFACTS is None: + self.fail("fixture must be built before running contract checks") + + def test_sqlite_v2_schema_and_package_metadata(self) -> None: + assert ARTIFACTS is not None + path = ARTIFACTS / "package-staging" / "Public" / "index.db" + with sqlite3.connect(path) as database: + self.assertEqual("ok", database.execute("PRAGMA integrity_check").fetchone()[0]) + tables = { + row[0] + for row in database.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + } + self.assertEqual( + { + "metadata", + "packages", + "tags2", + "tags2_map", + "commands2", + "commands2_map", + "pfns2", + "productcodes2", + "norm_names2", + "norm_publishers2", + "upgradecodes2", + }, + tables, + ) + metadata = dict(database.execute("SELECT name, value FROM metadata")) + self.assertEqual("2", metadata["majorVersion"]) + self.assertEqual("0", metadata["minorVersion"]) + self.assertRegex(metadata["databaseIdentifier"], r"^\{[0-9A-F-]{36}\}$") + self.assertTrue(metadata["lastwritetime"].isdigit()) + package = database.execute( + "SELECT id, name, latest_version, hash FROM packages WHERE rowid=1" + ).fetchone() + self.assertEqual( + ( + "Pinget.Contract.Singleton", + "Pinget Contract Fixture", + "1.0.0", + ), + package[:3], + ) + self.assertEqual(32, len(package[3])) + self.assertEqual( + [(1, 1)], + database.execute( + "SELECT tag, package FROM tags2_map" + ).fetchall(), + ) + + def test_sidecar_hash_matches_native_mszip_stream(self) -> None: + assert ARTIFACTS is not None + metadata = dict( + line.split("=", 1) + for line in (ARTIFACTS / "fixture-metadata.txt") + .read_text(encoding="utf-8") + .splitlines() + ) + package_hash = bytes.fromhex(metadata["SidecarSHA256"]) + expected_sidecar = ( + "packages/Pinget.Contract.Singleton/" + f"{metadata['SidecarSHA256'][:8].lower()}/versionData.mszyml" + ) + self.assertEqual(expected_sidecar, metadata["Sidecar"]) + sidecar = ( + ARTIFACTS + / "feed" + / Path(metadata["Sidecar"].replace("/", "\\")) + ).read_bytes() + self.assertEqual(package_hash, sha256(sidecar)) + self.assertEqual( + (ARTIFACTS / "versionData.yml").read_bytes(), + decompress_mszip(sidecar), + ) + with sqlite3.connect( + ARTIFACTS / "package-staging" / "Public" / "index.db" + ) as database: + index_hash = database.execute( + "SELECT hash FROM packages WHERE id=?", + ("Pinget.Contract.Singleton",), + ).fetchone()[0] + self.assertEqual(package_hash, index_hash) + self.assertTrue(set(metadata["SidecarSHA256"]) <= HASH_RE) + + def test_msix_contains_the_v2_index_and_hosted_artifacts(self) -> None: + assert ARTIFACTS is not None + package_path = ARTIFACTS / "feed" / "source2.msix" + with zipfile.ZipFile(package_path) as package: + names = set(package.namelist()) + self.assertIn("Public/index.db", names) + self.assertIn("AppxManifest.xml", names) + packaged_index = package.read("Public/index.db") + staged_index = ( + ARTIFACTS / "package-staging" / "Public" / "index.db" + ).read_bytes() + self.assertEqual(staged_index, packaged_index) + package_xml = ElementTree.fromstring( + package.read("AppxManifest.xml") + ) + app_extension = next( + node + for node in package_xml.iter() + if node.tag.endswith("AppExtension") + ) + self.assertEqual("com.microsoft.winget.source", app_extension.attrib["Name"]) + self.assertEqual("Public", app_extension.attrib["PublicFolder"]) + metadata = dict( + line.split("=", 1) + for line in (ARTIFACTS / "fixture-metadata.txt") + .read_text(encoding="utf-8") + .splitlines() + ) + self.assertNotIn(metadata["Manifest"], names) + self.assertNotIn(metadata["Sidecar"], names) + self.assertTrue((ARTIFACTS / "feed" / metadata["Manifest"]).is_file()) + self.assertTrue((ARTIFACTS / "feed" / metadata["Sidecar"]).is_file()) + self.assertEqual( + hashlib.sha256(PAYLOAD).hexdigest(), + metadata["InstallerSHA256"], + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--makeappx", type=Path, required=True) + parser.add_argument( + "--publisher", default="CN=Pinget Publisher Contract Test" + ) + parser.add_argument( + "--installer-url", + default="https://127.0.0.1:8765/HarmlessPayload.txt", + ) + args = parser.parse_args() + build(args.output, args.makeappx, args.publisher, args.installer_url) + result = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromTestCase(V2FixtureTests) + ) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/publisher/compare_production_snapshots.py b/tests/publisher/compare_production_snapshots.py new file mode 100644 index 0000000..3ef2e37 --- /dev/null +++ b/tests/publisher/compare_production_snapshots.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Compare two production snapshots by state identity and logical WinGet data.""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path +from typing import Any + +import yaml + +from build_v2_fixture import decompress_mszip + + +def canonical(value: Any) -> Any: + if isinstance(value, dict): + return {key: canonical(value[key]) for key in sorted(value)} + if isinstance(value, list): + return [canonical(item) for item in value] + return value + + +def snapshot_metadata(snapshot: Path) -> dict[str, Any]: + return json.loads((snapshot / "snapshot.json").read_text(encoding="utf-8")) + + +def read_index(snapshot: Path) -> dict[str, Any]: + database_path = snapshot / "Public" / "index.db" + with sqlite3.connect(f"file:{database_path.as_posix()}?mode=ro", uri=True) as db: + packages = {} + for package_id, name, moniker, version, arp_min, arp_max in db.execute( + "SELECT id,name,moniker,latest_version,arp_min_version,arp_max_version FROM packages" + ): + packages[package_id] = { + "name": name, + "moniker": moniker, + "latestVersion": version, + "arpMinimum": arp_min, + "arpMaximum": arp_max, + "tags": mapped_values(db, package_id, "tags2", "tag", "tags2_map", "tag"), + "commands": mapped_values(db, package_id, "commands2", "command", "commands2_map", "command"), + "pfns": correlation_values(db, package_id, "pfns2", "pfn"), + "productCodes": correlation_values(db, package_id, "productcodes2", "productcode"), + "upgradeCodes": correlation_values(db, package_id, "upgradecodes2", "upgradecode"), + "normalizedNames": correlation_values(db, package_id, "norm_names2", "norm_name"), + "normalizedPublishers": correlation_values(db, package_id, "norm_publishers2", "norm_publisher"), + } + return packages + + +def mapped_values( + db: sqlite3.Connection, + package_id: str, + value_table: str, + value_column: str, + map_table: str, + map_column: str, +) -> list[str]: + query = ( + f"SELECT valuesTable.{value_column} FROM {value_table} AS valuesTable " + f"JOIN {map_table} AS valueMap ON valueMap.{map_column}=valuesTable.rowid " + "JOIN packages AS package ON package.rowid=valueMap.package " + f"WHERE package.id=? ORDER BY valuesTable.{value_column}" + ) + return [row[0] for row in db.execute(query, (package_id,))] + + +def correlation_values( + db: sqlite3.Connection, package_id: str, table: str, column: str +) -> list[str]: + query = ( + f"SELECT value.{column} FROM {table} AS value " + "JOIN packages AS package ON package.rowid=value.package " + f"WHERE package.id=? ORDER BY value.{column}" + ) + return [row[0] for row in db.execute(query, (package_id,))] + + +def read_version_data(snapshot: Path) -> dict[str, Any]: + database_path = snapshot / "Public" / "index.db" + feed = snapshot / "feed" + with sqlite3.connect(f"file:{database_path.as_posix()}?mode=ro", uri=True) as db: + rows = db.execute("SELECT id,hash FROM packages").fetchall() + result: dict[str, Any] = {} + for package_id, package_hash in rows: + sidecar = ( + feed + / "packages" + / package_id + / package_hash.hex()[:8] + / "versionData.mszyml" + ) + document = yaml.safe_load( + decompress_mszip(sidecar.read_bytes()).decode("utf-8") + ) + if not isinstance(document, dict) or not isinstance(document.get("vD"), list): + raise AssertionError(f"invalid version sidecar for package {package_id}") + result[package_id] = canonical(document) + return result + + +def read_compressed_sidecars(snapshot: Path) -> dict[str, tuple[str, bytes]]: + database_path = snapshot / "Public" / "index.db" + feed = snapshot / "feed" + with sqlite3.connect(f"file:{database_path.as_posix()}?mode=ro", uri=True) as db: + rows = db.execute("SELECT id,hash FROM packages").fetchall() + result: dict[str, tuple[str, bytes]] = {} + for package_id, package_hash in rows: + digest = package_hash.hex() + sidecar = ( + feed + / "packages" + / package_id + / digest[:8] + / "versionData.mszyml" + ) + result[package_id] = (digest, sidecar.read_bytes()) + return result + + +def read_merged_manifests(snapshot: Path) -> dict[str, Any]: + feed = snapshot / "feed" + with sqlite3.connect(f"file:{(snapshot / 'Public' / 'index.db').as_posix()}?mode=ro", uri=True) as db: + hashes = [ + (package_id, package_hash.hex()) + for package_id, package_hash in db.execute("SELECT id,hash FROM packages") + ] + result: dict[str, Any] = {} + for package_id, package_hash in hashes: + sidecar = feed / "packages" / package_id / package_hash[:8] / "versionData.mszyml" + version_data = decompress_mszip(sidecar.read_bytes()).decode("utf-8") + records: set[str] = set() + for document in yaml.safe_load_all(version_data): + if not isinstance(document, dict): + continue + records.update( + item["rP"] + for item in document.get("vD", []) + if isinstance(item, dict) and isinstance(item.get("rP"), str) + ) + for relative in records: + manifest_path = feed / Path(relative.replace("/", "\\")) + for manifest in yaml.safe_load_all(manifest_path.read_text(encoding="utf-8")): + if not isinstance(manifest, dict): + continue + key = "|".join( + ( + str(manifest.get("PackageIdentifier", "")), + str(manifest.get("PackageVersion", "")), + str(manifest.get("PackageLocale", "")), + ) + ) + if key in result: + raise AssertionError(f"duplicate merged manifest identity: {key}") + result[key] = canonical(manifest) + return result + + +def read_merged_manifest_bytes(snapshot: Path) -> dict[str, bytes]: + result: dict[str, bytes] = {} + for path in (snapshot / "feed" / "manifests").rglob("*.yaml"): + raw = path.read_bytes() + document = yaml.safe_load(raw) + if not isinstance(document, dict): + raise AssertionError(f"merged manifest is not a mapping: {path}") + key = "|".join( + ( + str(document.get("PackageIdentifier", "")), + str(document.get("PackageVersion", "")), + str(document.get("PackageLocale", "")), + ) + ) + if key in result: + raise AssertionError(f"duplicate merged manifest identity: {key}") + result[key] = raw + return result + + +def compare(left: Path, right: Path) -> dict[str, Any]: + left = left.resolve(strict=True) + right = right.resolve(strict=True) + left_meta = snapshot_metadata(left) + right_meta = snapshot_metadata(right) + if left_meta.get("artifactFormatVersion") != 2 or right_meta.get("artifactFormatVersion") != 2: + raise AssertionError("both snapshots must use artifactFormatVersion 2") + if left_meta.get("buildId") != right_meta.get("buildId"): + raise AssertionError("Rust and .NET build identifiers differ for identical inputs") + if left_meta.get("inventory") != right_meta.get("inventory"): + raise AssertionError("canonical raw input inventories differ") + left_index = read_index(left) + right_index = read_index(right) + if canonical(left_index) != canonical(right_index): + raise AssertionError("WinGet logical database rows or correlation mappings differ") + left_sidecars = read_compressed_sidecars(left) + right_sidecars = read_compressed_sidecars(right) + if left_sidecars != right_sidecars: + differing = sorted( + package_id + for package_id in left_sidecars.keys() | right_sidecars.keys() + if left_sidecars.get(package_id) != right_sidecars.get(package_id) + ) + raise AssertionError( + "compressed version sidecar bytes or package hash differ: " + + ", ".join(differing) + ) + left_version_data = read_version_data(left) + right_version_data = read_version_data(right) + if canonical(left_version_data) != canonical(right_version_data): + raise AssertionError("WinGet version sidecar data differs between implementations") + left_manifests = read_merged_manifests(left) + right_manifests = read_merged_manifests(right) + if canonical(left_manifests) != canonical(right_manifests): + raise AssertionError("canonical merged manifest data differs between implementations") + left_manifest_bytes = read_merged_manifest_bytes(left) + right_manifest_bytes = read_merged_manifest_bytes(right) + if left_manifest_bytes != right_manifest_bytes: + differing = sorted( + key + for key in left_manifest_bytes.keys() | right_manifest_bytes.keys() + if left_manifest_bytes.get(key) != right_manifest_bytes.get(key) + ) + raise AssertionError("raw merged manifest bytes differ: " + ", ".join(differing)) + return { + "buildId": left_meta["buildId"], + "packages": len(left_index), + "manifests": len(left_manifests), + "logicalIndex": "equal", + "compressedSidecarBytes": "equal", + "versionData": "equal", + "canonicalManifests": "equal", + "rawMergedManifestBytes": "equal", + "rawInputInventory": "equal", + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--rust-snapshot", type=Path, required=True) + parser.add_argument("--dotnet-snapshot", type=Path, required=True) + args = parser.parse_args() + try: + print( + json.dumps( + compare(args.rust_snapshot, args.dotnet_snapshot), + sort_keys=True, + ) + ) + except Exception as error: + print(f"cross-language snapshot comparison failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/publisher/contract.md b/tests/publisher/contract.md new file mode 100644 index 0000000..0cad249 --- /dev/null +++ b/tests/publisher/contract.md @@ -0,0 +1,175 @@ +# WinGet v2 publisher compatibility evidence + +This file records the independently derived v2 source-package contract for the +publisher compatibility gate. The evidence is pinned to read-only upstream +WinGet commit `cf71e38eccd2b9035c03c7408f5328cd0e4fdee8`; it is not a runtime or +build dependency. + +## Database package layout + +A v2 source is served from a base URL. WinGet requests `source2.msix`; its +package manifest exposes `Public`, and the packaged database must be +`Public/index.db`. The same base URL serves external package artifacts. +`WinGetSourceCreator/WinGetLocalSource.cs` emits this path and package layout. + +`index-schema.sql` captures the published data tables retained after +`Microsoft/Schema/2_0/Interface_2_0.cpp::PrepareForPackaging`. In particular: + +- The `metadata` table has the fixed WinGet shape: `name TEXT PRIMARY KEY NOT + NULL`, `value TEXT NOT NULL`, `WITHOUT ROWID`. +- `packages` has an integer `rowid` primary key followed by `id`, `name`, + nullable `moniker`, `latest_version`, nullable ARP bounds, and nullable + `hash BLOB`. The package identifier is not the SQL primary key. +- `tags2` and `commands2` use integer-rowid value tables and `*_map` tables + keyed by `(value-rowid, package-rowid)`. +- Correlation tables `pfns2`, `productcodes2`, `norm_names2`, + `norm_publishers2`, and `upgradecodes2` use `(value, package)` composite + primary keys without rowids. +- Writer-side indexes, internal v1 tables, and package-update tracking tables + are removed during packaging. The candidate schema was corrected to reflect + those retained shapes; it must not be interpreted as the schema of the + private mutable writer database. + +The required schema markers are `majorVersion=2` and `minorVersion=0` in +`metadata`. Upstream creates a `databaseIdentifier` GUID and writes +`lastwritetime`; both are included in the fixture. The SQLite `user_version` +pragma is not the WinGet schema marker. + +## Version data and hosted manifest references + +Each package row's `hash` is the 32-byte SHA-256 of the exact compressed +version-data bytes. The lowercase hex form identifies the sidecar directory: +`packages///versionData.mszyml`. +`PackageVersionDataManifest.cpp` defines the YAML keys `sV`, `vD`, `v`, +`aMiV`, `aMaV`, `rP`, and `s256H`, with schema version `"1.0"`. Each version +references a hosted manifest by relative URL path and lowercase hexadecimal +SHA-256 of its exact bytes. The upstream path construction retains the package +identifier and takes the first eight lowercase hash characters. + +Upstream calls Windows Compression API `CreateCompressor` with +`COMPRESS_ALGORITHM_MSZIP` and stores the bytes returned by `Compress` directly; +there is no additional application framing in `Compression.cpp`. The fixture +generator uses that Windows API and verifies the result with +`CreateDecompressor`/`Decompress`, then hashes the bytes actually stored. + +The exact manifest serialization, version ordering, and search-normalization +values are not inferred from the schema files. The locally authored singleton +fixture was accepted by stock WinGet 1.29.380 in a fresh Windows Sandbox: +source add/update, exact-ID search, show of the hosted manifest, and download +of the harmless payload all succeeded. The downloaded payload's bytes and +SHA-256 matched the locally authored fixture; it was not installed. This +acceptance covers the singleton full-index case only, not delta behavior, +multiple-version ordering, or broad search-normalization equivalence. + +## Test fixture and independent acceptance + +The fixture is locally authored and deliberately harmless. It is generated +under ignored `artifacts/publisher-contract/` (or an external session path), +never under the source fixture directory. `source2.msix`, the index, sidecar, +signing certificate, payload, and server logs are generated test output, not +committed source assets. + +The required independent gate is a fresh Windows Sandbox with stock WinGet. +Trust for the ephemeral signing certificate and the test source registration +are confined to that Sandbox. WinGet's package deployment path required the +fixture signer in the Sandbox machine `Root` store; trusting it only in the +Sandbox user's `TrustedPeople`/`Root` stores was insufficient (`0x800B0109`). +The host certificate stores were not changed. The test adds/updates the source, +searches and shows the fixture package, then downloads the harmless payload +and verifies its expected SHA-256 without installing it. A Python SQLite read +and native MSZIP round trip establish additional independent checks, but +neither substitutes for the stock client. + +## Non-upstream publisher policy (separate from the independent fixture gate) + +The build-state schema, build IDs, manifest canonicalization, snapshot layout, +CLI switches, ownership marker, atomic promotion behavior, and cross-language +state interoperability are Pinget design decisions, not WinGet compatibility +requirements. They are validated separately by the production-publisher +conformance suite, not by the independent fixture gate above. + +State database schema is an implementation detail validated through the +production writers and readers. Rust embeds its state schema in the publisher +library; the conformance suite cross-opens and verifies state created by each +implementation rather than maintaining a separate test-side schema copy. + +## Pinget publisher command contract + +The work directory is always selected with `--state`; `--output` remains the +global result format selector (`text`, `json`, or `yaml`) and must never be +used as a state-directory alias. + +```powershell +pinget repository build --input --state +pinget repository diff --input --state +pinget repository status --state +pinget repository verify --state [--build ] +pinget repository verify --destination +pinget repository doctor +pinget repository publish --state --destination ` + --name --publisher --version ` + (--pfx --password-env | --unsigned-development) +``` + +The `build` command creates a snapshot; it does not publish or sign an MSIX. +The `publish` command consumes that snapshot and uses `source2.msix` as the +published source-package path. Test harnesses may pass a built CLI executable +or an already-built .NET CLI assembly, but must preserve these argument +meanings. This command contract is a Pinget implementation expectation and is +not part of the upstream WinGet format. + +## Production publisher cross-language contract (tested separately) + +The current publisher conformance gate exercises the actual Rust and .NET +writers/readers. Both implementations create immutable snapshots at +`/builds//feed/`; the build +snapshot contains the feed artifacts and index plus `snapshot.json`. State +commands are read-only when inspecting missing/uninitialized state. +Equivalent manifest trees yield matching build identities, and Rust and .NET +can each reopen and verify state created by the other. + +The current publisher artifact format is version 2, recorded as +`artifactFormatVersion=2` in private state metadata and as +`artifactFormatVersion: 2` in `snapshot.json`. The SQLite `user_version` and +structural `schemaVersion` remain 1. Build identity uses the +`pinget-publisher-state-v2\n` salt. Legacy state or snapshots without the +current artifact-format marker are rejected without mutation; use a fresh +`--state` directory rather than relying on automatic migration. The existing +destination ownership marker remains independently versioned by its existing +schema and does not require this snapshot-format field. + +Cross-language comparisons pin canonical manifest bytes, raw input inventory, +logical SQLite index contents, version-data sidecars and their hashes, and the +snapshot `indexHash`. They also validate the resulting SQLite tables with +Python's `sqlite3` and the sidecar bytes with the Windows native MSZIP API. +These checks establish the tested outputs and interoperability; the private +state schema remains governed by the embedded production schemas. + +In JSON mode, the tested production command reports use these shared field +sets and shapes (nullable values are `null`; command failures are nonzero and +do not return a success report): + +| Command | JSON fields | +| --- | --- | +| `repository build` | `buildId`, `snapshotDirectory`, `manifestCount`, `packageCount`, `changedFiles`, `removedFiles` | +| `repository diff` | `lastBuildId`, `published`, `added`, `changed`, `removed` | +| `repository status` | `lastBuildId`, `lastPublishedBuildId`, `buildCount`, `lastBuildTimestamp` | +| `repository verify` / successful `repository publish` | `valid`, `subject`, `artifactCount`, `buildId` | +| `repository doctor` | `operatingSystem`, `makeAppxPath`, `signToolPath`, `nativeMsZipAvailable` | + +`doctor.operatingSystem` is informational and is intentionally not compared +between platforms/implementations. `snapshotDirectory` is absolute. Build +report `changedFiles` counts new or modified input files relative to the prior +build (zero on the first or unchanged build); `removedFiles` counts removed +inputs. Destination verification uses `.pinget-publisher.json` as the +ownership record, checks the published MSIX/index/identity and exact artifact +inventory, and rejects untracked destination files rather than adopting or +deleting them. The marker and private state DB are Pinget contracts, not +WinGet-defined structures. + +The producer emits Windows Compression API MSZIP framing. Native independent +validation and core reader regression tests cover sidecars spanning multiple +32 KiB blocks, including the version-data record after the first block; this +is separate from the singleton stock-WinGet fixture acceptance above. Legacy +single-block `CK`/raw-deflate input remains a separate reader compatibility +path. diff --git a/tests/publisher/fixtures/AppxManifest.xml.in b/tests/publisher/fixtures/AppxManifest.xml.in new file mode 100644 index 0000000..edd2b9a --- /dev/null +++ b/tests/publisher/fixtures/AppxManifest.xml.in @@ -0,0 +1,43 @@ + + + + + Pinget Publisher Contract Fixture + Pinget Test Fixtures + Assets\StoreLogo.png + + + + + + + + + + + + + + + + + + + + diff --git a/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.installer.yaml b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.installer.yaml new file mode 100644 index 0000000..4e2d529 --- /dev/null +++ b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.installer.yaml @@ -0,0 +1,37 @@ +PackageIdentifier: Pinget.Test.Matrix +PackageVersion: 1.10.0 +Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://example.invalid/matrix-1.10.0.exe + InstallerSha256: 1111111111111111111111111111111111111111111111111111111111111111 + Scope: user + InstallerSwitches: + Silent: /quiet + SilentWithProgress: /quiet-progress + Interactive: /interactive + Dependencies: + PackageDependencies: + - PackageIdentifier: Pinget.Test.Runtime + MinimumVersion: 3.2.0 + PackageFamilyName: pinget.matrix_1234567890abc + Commands: + - OtherCmd + - othercmd + - matrix + - '' + ProductCode: '{00112233-4455-6677-8899-aabbccddeeff}' + UpgradeCode: '{11223344-5566-7788-99aa-bbccddeeff00}' + AppsAndFeaturesEntries: + - DisplayName: Matrix Desktop + Publisher: Pinget Test Fixtures + DisplayVersion: 1.10.0 + ProductCode: '{00112233-4455-6677-8899-AABBCCDDEEFF}' + UpgradeCode: '{11223344-5566-7788-99AA-BBCCDDEEFF00}' + - Architecture: x64 + InstallerType: portable + InstallerUrl: https://example.invalid/matrix-portable-1.10.0.zip + InstallerSha256: 2222222222222222222222222222222222222222222222222222222222222222 + PortableCommandAlias: matrix-portable +ManifestType: installer +ManifestVersion: 1.9.0 diff --git a/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.en-US.yaml b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.en-US.yaml new file mode 100644 index 0000000..7bd031e --- /dev/null +++ b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.en-US.yaml @@ -0,0 +1,28 @@ +PackageIdentifier: Pinget.Test.Matrix +PackageVersion: 1.10.0 +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PublisherUrl: https://example.invalid/ +PackageName: Matrix Package +PackageFamilyName: Pinget.Matrix_1234567890abc +GenericNumber: 1.0 +Small: 0.5 +Tiny: 1e-6 +Threshold: 1e-5 +Large: 1e16 +NegativeZero: -0.0 +MaxUInt: 18446744073709551615 +License: MIT +ShortDescription: Multi-file matrix package with installer and correlation metadata +Tags: + - publisher-conformance + - matrix + - Matrix + - '' +Commands: + - matrix + - matrix-cli + - Matrix + - '' +ManifestType: defaultLocale +ManifestVersion: 1.9.0 diff --git a/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml new file mode 100644 index 0000000..a8cd918 --- /dev/null +++ b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml @@ -0,0 +1,10 @@ +PackageIdentifier: Pinget.Test.Matrix +PackageVersion: 1.10.0 +PackageLocale: fr-FR +Publisher: Pinget Localization Lab +PackageName: Matrice +ShortDescription: Fixture localisee pour verifier les manifestes de langue +Tags: + - matrice +ManifestType: locale +ManifestVersion: 1.9.0 diff --git a/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.version.yaml b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.version.yaml new file mode 100644 index 0000000..8f8fc1f --- /dev/null +++ b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.10.0.version.yaml @@ -0,0 +1,5 @@ +PackageIdentifier: Pinget.Test.Matrix +PackageVersion: 1.10.0 +DefaultLocale: en-US +ManifestType: version +ManifestVersion: 1.9.0 diff --git a/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.9.0.yaml b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.9.0.yaml new file mode 100644 index 0000000..7b76c22 --- /dev/null +++ b/tests/publisher/fixtures/multi/Pinget.Test.Matrix_1.9.0.yaml @@ -0,0 +1,17 @@ +PackageIdentifier: Pinget.Test.Matrix +PackageVersion: 1.9.0 +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PublisherUrl: https://example.invalid/ +PackageName: Matrix Package +License: MIT +ShortDescription: Previous version used to distinguish numeric version ordering +AppsAndFeaturesEntries: + - DisplayVersion: 1.9.0 +Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://example.invalid/matrix-1.9.0.exe + InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 +ManifestType: singleton +ManifestVersion: 1.9.0 diff --git a/tests/publisher/fixtures/singleton.yaml.in b/tests/publisher/fixtures/singleton.yaml.in new file mode 100644 index 0000000..b82d807 --- /dev/null +++ b/tests/publisher/fixtures/singleton.yaml.in @@ -0,0 +1,19 @@ +PackageIdentifier: Pinget.Contract.Singleton +PackageVersion: 1.0.0 +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PublisherUrl: https://example.invalid/ +PackageName: Pinget Contract Fixture +License: MIT +ShortDescription: Harmless locally authored publisher compatibility fixture +Moniker: pinget-contract +Tags: + - publisher-contract +Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: {{INSTALLER_URL}} + InstallerSha256: {{INSTALLER_SHA256}} + Scope: user +ManifestType: singleton +ManifestVersion: 1.9.0 diff --git a/tests/publisher/fixtures/singleton/Pinget.Contract.Singleton.yaml b/tests/publisher/fixtures/singleton/Pinget.Contract.Singleton.yaml new file mode 100644 index 0000000..1dc9f12 --- /dev/null +++ b/tests/publisher/fixtures/singleton/Pinget.Contract.Singleton.yaml @@ -0,0 +1,19 @@ +PackageIdentifier: Pinget.Contract.Singleton +PackageVersion: 1.0.0 +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PublisherUrl: https://example.invalid/ +PackageName: Pinget Contract Fixture +License: MIT +ShortDescription: Harmless production publisher conformance fixture +Moniker: pinget-contract +Tags: + - publisher-contract +Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://127.0.0.1:8765/HarmlessPayload.txt + InstallerSha256: 96c3a679ab1dc50990f84b7610f41bad666d92a82aceaf2edb8674a53587f5b7 + Scope: user +ManifestType: singleton +ManifestVersion: 1.9.0 diff --git a/tests/publisher/index-schema.sql b/tests/publisher/index-schema.sql new file mode 100644 index 0000000..4a8091d --- /dev/null +++ b/tests/publisher/index-schema.sql @@ -0,0 +1,66 @@ +-- WinGet schema 2.0 data tables retained by Interface::PrepareForPackaging. +-- Source: upstream winget-cli cf71e38eccd2b9035c03c7408f5328cd0e4fdee8, +-- AppInstallerRepositoryCore/Microsoft/Schema/2_0 and +-- AppInstallerSharedLib/SQLiteMetadataTable.cpp. +-- Preparation drops the writer-side indexes and all internal v1 tables. +CREATE TABLE metadata( + name TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE packages( + rowid INTEGER PRIMARY KEY, + id TEXT NOT NULL, + name TEXT NOT NULL, + moniker TEXT, + latest_version TEXT NOT NULL, + arp_min_version TEXT, + arp_max_version TEXT, + hash BLOB +); + +CREATE TABLE tags2( + rowid INTEGER PRIMARY KEY, + tag TEXT NOT NULL +); +CREATE TABLE tags2_map( + tag INTEGER NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(tag, package) +) WITHOUT ROWID; + +CREATE TABLE commands2( + rowid INTEGER PRIMARY KEY, + command TEXT NOT NULL +); +CREATE TABLE commands2_map( + command INTEGER NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(command, package) +) WITHOUT ROWID; + +CREATE TABLE pfns2( + pfn TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(pfn, package) +) WITHOUT ROWID; +CREATE TABLE productcodes2( + productcode TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(productcode, package) +) WITHOUT ROWID; +CREATE TABLE norm_names2( + norm_name TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(norm_name, package) +) WITHOUT ROWID; +CREATE TABLE norm_publishers2( + norm_publisher TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(norm_publisher, package) +) WITHOUT ROWID; +CREATE TABLE upgradecodes2( + upgradecode TEXT NOT NULL, + package INTEGER NOT NULL, + PRIMARY KEY(upgradecode, package) +) WITHOUT ROWID; diff --git a/tests/publisher/run-production-conformance.ps1 b/tests/publisher/run-production-conformance.ps1 new file mode 100644 index 0000000..a052c9f --- /dev/null +++ b/tests/publisher/run-production-conformance.ps1 @@ -0,0 +1,285 @@ +param( + [string] $RustCli = (Join-Path $PSScriptRoot '..\..\rust\target\release\pinget.exe'), + [string] $DotnetAssembly = (Join-Path $PSScriptRoot '..\..\dotnet\src\Devolutions.Pinget.Cli\bin\Release\net10.0\pinget.dll'), + [string] $DotnetExecutable = '', + [string] $DotnetHost = 'dotnet', + [string] $InputDirectory = (Join-Path $PSScriptRoot 'fixtures\multi'), + [string] $WorkDirectory = (Join-Path $env:TEMP ('PingetPublisherConformance-' + [Guid]::NewGuid().ToString('N'))), + [string] $ExpectedPackageId = 'Pinget.Test.Matrix', + [string] $Python = 'python', + [string] $MakeAppxPath = 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64\makeappx.exe', + [string] $PackageName = 'Pinget.Publisher.Conformance', + [string] $Publisher = 'CN=Pinget Publisher Conformance Test' +) + +$ErrorActionPreference = 'Stop' +$repositoryRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..')) +$inputPath = [System.IO.Path]::GetFullPath($InputDirectory) +$workPath = [System.IO.Path]::GetFullPath($WorkDirectory) +$rustPath = [System.IO.Path]::GetFullPath($RustCli) +$dotnetPath = [System.IO.Path]::GetFullPath($DotnetAssembly) +$dotnetExecutablePath = if ([string]::IsNullOrWhiteSpace($DotnetExecutable)) { + '' +} else { + [System.IO.Path]::GetFullPath($DotnetExecutable) +} +$validator = Join-Path $PSScriptRoot 'validate_production_snapshot.py' +$comparer = Join-Path $PSScriptRoot 'compare_production_snapshots.py' + +foreach ($required in @($inputPath, $rustPath, $validator, $comparer, $MakeAppxPath)) { + if (-not (Test-Path -LiteralPath $required)) { + throw "Required publisher conformance input does not exist: $required" + } +} +if ([string]::IsNullOrWhiteSpace($dotnetExecutablePath) -and -not (Test-Path -LiteralPath $dotnetPath)) { + throw "Required publisher conformance input does not exist: $dotnetPath" +} +if (-not [string]::IsNullOrWhiteSpace($dotnetExecutablePath) -and + -not (Test-Path -LiteralPath $dotnetExecutablePath)) { + throw "Required publisher conformance input does not exist: $dotnetExecutablePath" +} +New-Item -ItemType Directory -Force -Path $workPath | Out-Null +$rustState = Join-Path $workPath 'rust-state' +$dotnetState = Join-Path $workPath 'dotnet-state' + +function Invoke-Pinget { + param( + [ValidateSet('Rust', 'Dotnet')] + [string] $Implementation, + [string[]] $Arguments + ) + Write-Host ("RUN {0}: {1}" -f $Implementation, ($Arguments -join ' ')) + if ($Implementation -eq 'Rust') { + $result = & $script:rustPath @Arguments 2>&1 + } + elseif (-not [string]::IsNullOrWhiteSpace($script:dotnetExecutablePath)) { + $result = & $script:dotnetExecutablePath @Arguments 2>&1 + } + else { + $result = & $script:DotnetHost $script:dotnetPath @Arguments 2>&1 + } + $code = $LASTEXITCODE + $text = ($result | ForEach-Object { $_.ToString() }) -join "`n" + if ($code -ne 0) { + throw "$Implementation CLI exited with $code for '$($Arguments -join ' ')':`n$text" + } + return $text +} + +function Invoke-PingetRaw { + param( + [ValidateSet('Rust', 'Dotnet')] + [string] $Implementation, + [string[]] $Arguments + ) + if ($Implementation -eq 'Rust') { + $result = & $script:rustPath @Arguments 2>&1 + } + elseif (-not [string]::IsNullOrWhiteSpace($script:dotnetExecutablePath)) { + $result = & $script:dotnetExecutablePath @Arguments 2>&1 + } + else { + $result = & $script:DotnetHost $script:dotnetPath @Arguments 2>&1 + } + return [pscustomobject]@{ + ExitCode = $LASTEXITCODE + Text = ($result | ForEach-Object { $_.ToString() }) -join "`n" + } +} + +function ConvertFrom-PingetJson { + param([string] $Text, [string] $Context) + try { + return $Text | ConvertFrom-Json + } + catch { + throw "$Context did not return valid JSON: $Text" + } +} + +function Assert-ReportFields { + param([object] $Report, [string[]] $ExpectedFields, [string] $Context) + $actual = @($Report.PSObject.Properties.Name | Sort-Object) + $expected = @($ExpectedFields | Sort-Object) + if (($actual -join "`n") -ne ($expected -join "`n")) { + throw "$Context JSON fields differ. Expected [$($expected -join ', ')], got [$($actual -join ', ')]." + } +} + +function Assert-VerifiedReport { + param([object] $Report, [string] $BuildId, [string] $Context) + Assert-ReportFields $Report @('valid', 'subject', 'artifactCount', 'buildId') $Context + if (-not $Report.valid -or $Report.buildId -ne $BuildId -or $Report.artifactCount -le 0) { + throw "$Context did not verify build ${BuildId}: $($Report | ConvertTo-Json -Compress)" + } +} + +function Get-BuildId { + param([string] $StateDirectory) + $builds = Join-Path $StateDirectory 'builds' + if (-not (Test-Path -LiteralPath $builds)) { + throw "Publisher state did not create its builds directory: $builds" + } + $ids = @(Get-ChildItem -LiteralPath $builds -Directory | + Where-Object { $_.Name -match '^[0-9a-f]{64}$' } | + Select-Object -ExpandProperty Name) + if ($ids.Count -ne 1) { + throw "Expected one build snapshot under $builds; found $($ids.Count)." + } + return $ids[0] +} + +$common = @('--output', 'json', 'repository') +$rustBuildOutput = Invoke-Pinget 'Rust' ($common + @('build', '--input', $inputPath, '--state', $rustState)) +$dotnetBuildOutput = Invoke-Pinget 'Dotnet' ($common + @('build', '--input', $inputPath, '--state', $dotnetState)) +$rustBuildReport = ConvertFrom-PingetJson $rustBuildOutput 'Rust build' +$dotnetBuildReport = ConvertFrom-PingetJson $dotnetBuildOutput '.NET build' +Assert-ReportFields $rustBuildReport @('buildId', 'snapshotDirectory', 'manifestCount', 'packageCount', 'changedFiles', 'removedFiles') 'Rust build' +Assert-ReportFields $dotnetBuildReport @('buildId', 'snapshotDirectory', 'manifestCount', 'packageCount', 'changedFiles', 'removedFiles') '.NET build' +$rustBuildId = Get-BuildId $rustState +$dotnetBuildId = Get-BuildId $dotnetState +if ($rustBuildId -ne $dotnetBuildId) { + throw "Cross-language build IDs differ: Rust=$rustBuildId .NET=$dotnetBuildId" +} +if ($rustBuildReport.buildId -ne $rustBuildId -or $dotnetBuildReport.buildId -ne $dotnetBuildId) { + throw 'Build JSON identifiers did not match the created snapshots.' +} +foreach ($buildReport in @($rustBuildReport, $dotnetBuildReport)) { + if ($buildReport.packageCount -ne 1 -or $buildReport.manifestCount -ne 2 -or + $buildReport.changedFiles -ne 0 -or $buildReport.removedFiles -ne 0) { + throw "Unexpected first-build summary: $($buildReport | ConvertTo-Json -Compress)" + } +} + +foreach ($implementation in @('Rust', 'Dotnet')) { + $statePath = if ($implementation -eq 'Rust') { $rustState } else { $dotnetState } + foreach ($format in @('json', 'yaml', 'text')) { + $formatted = Invoke-PingetRaw $implementation @( + '--output', $format, 'repository', 'status', '--state', $statePath + ) + if ($formatted.ExitCode -ne 0) { + throw "$implementation status --output $format failed ($($formatted.ExitCode)): $($formatted.Text)" + } + switch ($format) { + 'json' { + $formattedStatus = ConvertFrom-PingetJson $formatted.Text "$implementation JSON status" + if ($formattedStatus.lastBuildId -ne $rustBuildId) { + throw "$implementation JSON status omitted the current build id." + } + } + 'yaml' { + if ($formatted.Text -notmatch "(?m)^lastBuildId:\s*['""]?$([regex]::Escape($rustBuildId))['""]?\s*$") { + throw "$implementation YAML status omitted the current build id: $($formatted.Text)" + } + } + 'text' { + if ($formatted.Text -notmatch [regex]::Escape($rustBuildId)) { + throw "$implementation text status omitted the current build id: $($formatted.Text)" + } + } + } + } + $invalidFormat = Invoke-PingetRaw $implementation @( + '--output', 'xml', 'repository', 'status', '--state', $statePath + ) + if ($invalidFormat.ExitCode -eq 0) { + throw "$implementation accepted unsupported --output xml." + } + Write-Output "PASS Test_PublisherCli_$($implementation)_OutputFormatParity" +} + +$dotnetStatusOnRust = ConvertFrom-PingetJson ` + (Invoke-Pinget 'Dotnet' ($common + @('status', '--state', $rustState))) '.NET status on Rust state' +$rustStatusOnDotnet = ConvertFrom-PingetJson ` + (Invoke-Pinget 'Rust' ($common + @('status', '--state', $dotnetState))) 'Rust status on .NET state' +foreach ($statusReport in @($dotnetStatusOnRust, $rustStatusOnDotnet)) { + Assert-ReportFields $statusReport @('lastBuildId', 'lastPublishedBuildId', 'buildCount', 'lastBuildTimestamp') 'status' + if ($statusReport.lastBuildId -ne $rustBuildId -or $statusReport.lastPublishedBuildId -ne $null -or + $statusReport.buildCount -ne 1 -or [string]::IsNullOrWhiteSpace($statusReport.lastBuildTimestamp)) { + throw "Unexpected cross-open status report: $($statusReport | ConvertTo-Json -Compress)" + } +} +$dotnetVerifyRust = ConvertFrom-PingetJson ` + (Invoke-Pinget 'Dotnet' ($common + @('verify', '--state', $rustState, '--build', $rustBuildId))) '.NET verify of Rust snapshot' +$rustVerifyDotnet = ConvertFrom-PingetJson ` + (Invoke-Pinget 'Rust' ($common + @('verify', '--state', $dotnetState, '--build', $dotnetBuildId))) 'Rust verify of .NET snapshot' +Assert-VerifiedReport $dotnetVerifyRust $rustBuildId '.NET verify of Rust snapshot' +Assert-VerifiedReport $rustVerifyDotnet $dotnetBuildId 'Rust verify of .NET snapshot' + +$rustDoctor = ConvertFrom-PingetJson (Invoke-Pinget 'Rust' ($common + @('doctor'))) 'Rust doctor' +$dotnetDoctor = ConvertFrom-PingetJson (Invoke-Pinget 'Dotnet' ($common + @('doctor'))) '.NET doctor' +foreach ($doctorReport in @($rustDoctor, $dotnetDoctor)) { + Assert-ReportFields $doctorReport @('operatingSystem', 'makeAppxPath', 'signToolPath', 'nativeMsZipAvailable') 'doctor' + if ([string]::IsNullOrWhiteSpace($doctorReport.operatingSystem) -or -not $doctorReport.nativeMsZipAvailable) { + throw "Unexpected doctor report: $($doctorReport | ConvertTo-Json -Compress)" + } +} + +$rustSnapshot = Join-Path (Join-Path $rustState 'builds') $rustBuildId +$dotnetSnapshot = Join-Path (Join-Path $dotnetState 'builds') $dotnetBuildId +foreach ($snapshot in @($rustSnapshot, $dotnetSnapshot)) { + & $Python -B $validator --snapshot $snapshot --expect-package $ExpectedPackageId + if ($LASTEXITCODE -ne 0) { + throw "Independent SQLite/MSZIP snapshot validation failed for $snapshot (exit $LASTEXITCODE)." + } +} +& $Python -B $comparer --rust-snapshot $rustSnapshot --dotnet-snapshot $dotnetSnapshot +if ($LASTEXITCODE -ne 0) { + throw "Cross-language logical snapshot comparison failed (exit $LASTEXITCODE)." +} + +$rustDiff = ConvertFrom-PingetJson ` + (Invoke-Pinget 'Rust' ($common + @('diff', '--input', $inputPath, '--state', $rustState))) 'Rust diff' +$dotnetDiff = ConvertFrom-PingetJson ` + (Invoke-Pinget 'Dotnet' ($common + @('diff', '--input', $inputPath, '--state', $dotnetState))) '.NET diff' +foreach ($diffReport in @($rustDiff, $dotnetDiff)) { + Assert-ReportFields $diffReport @('lastBuildId', 'published', 'added', 'changed', 'removed') 'diff' + if ($diffReport.lastBuildId -ne $rustBuildId -or $diffReport.published -or + @($diffReport.added).Count -ne 0 -or @($diffReport.changed).Count -ne 0 -or + @($diffReport.removed).Count -ne 0) { + throw "Unexpected unchanged diff report: $($diffReport | ConvertTo-Json -Compress)" + } +} + +$rustDestination = Join-Path $workPath 'rust-unsigned-feed' +$dotnetDestination = Join-Path $workPath 'dotnet-unsigned-feed' +$rustPublish = ConvertFrom-PingetJson (Invoke-Pinget 'Rust' ($common + @( + 'publish', '--state', $rustState, '--destination', $rustDestination, + '--name', $PackageName, '--publisher', $Publisher, '--version', '1.0.0.0', + '--unsigned-development', '--makeappx', $MakeAppxPath +))) 'Rust publish' +$dotnetPublish = ConvertFrom-PingetJson (Invoke-Pinget 'Dotnet' ($common + @( + 'publish', '--state', $dotnetState, '--destination', $dotnetDestination, + '--name', $PackageName, '--publisher', $Publisher, '--version', '1.0.0.0', + '--unsigned-development', '--makeappx', $MakeAppxPath +))) '.NET publish' +Assert-VerifiedReport $rustPublish $rustBuildId 'Rust publish' +Assert-VerifiedReport $dotnetPublish $dotnetBuildId '.NET publish' +if ($rustPublish.subject -ne $dotnetPublish.subject -or + $rustPublish.artifactCount -ne $dotnetPublish.artifactCount) { + throw 'Rust and .NET publish reports differ in verified subject or artifact count.' +} +$dotnetVerifyRustDestination = ConvertFrom-PingetJson (Invoke-Pinget 'Dotnet' ($common + @( + 'verify', '--destination', $rustDestination +))) '.NET destination verify of Rust output' +$rustVerifyDotnetDestination = ConvertFrom-PingetJson (Invoke-Pinget 'Rust' ($common + @( + 'verify', '--destination', $dotnetDestination +))) 'Rust destination verify of .NET output' +Assert-VerifiedReport $dotnetVerifyRustDestination $rustBuildId '.NET destination verify of Rust output' +Assert-VerifiedReport $rustVerifyDotnetDestination $dotnetBuildId 'Rust destination verify of .NET output' + +[pscustomobject]@{ + Result = 'PASS' + BuildId = $rustBuildId + RustSnapshot = $rustSnapshot + DotnetSnapshot = $dotnetSnapshot + RustDestination = $rustDestination + DotnetDestination = $dotnetDestination + WorkDirectory = $workPath + RequiredArtifacts = @( + 'Rust and .NET status/verify can reopen each other''s state' + 'Python sqlite3 and native Windows MSZIP validation passed for both snapshots' + 'Canonicalized package, correlation, manifest and input-inventory data matched' + 'Rust and .NET reciprocal destination verification passed for unsigned MSIX outputs' + ) +} | ConvertTo-Json -Depth 4 diff --git a/tests/publisher/run-sandbox-publisher-gate.ps1 b/tests/publisher/run-sandbox-publisher-gate.ps1 new file mode 100644 index 0000000..1978df4 --- /dev/null +++ b/tests/publisher/run-sandbox-publisher-gate.ps1 @@ -0,0 +1,177 @@ +param( + [Parameter(Mandatory = $true)] + [string] $StageRoot, + [Parameter(Mandatory = $true)] + [string] $ResultsRoot +) + +$ErrorActionPreference = 'Stop' +$StageRoot = [System.IO.Path]::GetFullPath($StageRoot) +$ResultsRoot = [System.IO.Path]::GetFullPath($ResultsRoot) +$bootstrapRoot = Join-Path $ResultsRoot 'bootstrap' +$transcriptPath = Join-Path $ResultsRoot 'publisher-gate-transcript.txt' +$summaryPath = Join-Path $ResultsRoot 'publisher-gate-summary.json' +$signerCertificatePath = Join-Path $StageRoot 'publisher-contract.cer' +$pfxPath = Join-Path $StageRoot 'publisher-contract.pfx' +$passwordPath = Join-Path $StageRoot 'publisher-pfx-password.txt' +$testRootThumbprints = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::OrdinalIgnoreCase +) +$runSummary = [System.Collections.Generic.List[object]]::new() +$exitCode = 0 + +if (-not (Test-Path -LiteralPath $ResultsRoot)) { + throw "Writable Sandbox results folder is missing: $ResultsRoot" +} +if (-not (Test-Path -LiteralPath $signerCertificatePath) -or + -not (Test-Path -LiteralPath $pfxPath) -or + -not (Test-Path -LiteralPath $passwordPath)) { + throw 'The read-only staged signer certificate, PFX, and password file are required.' +} + +$signer = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new( + $signerCertificatePath) +$testRootThumbprints.Add($signer.Thumbprint) | Out-Null +$signer.Dispose() + +try { + Start-Transcript -LiteralPath $transcriptPath -Force | Out-Null + $env:PINGET_PUBLISHER_PFX_PASSWORD = [System.IO.File]::ReadAllText($passwordPath) + + Copy-Item -LiteralPath (Join-Path $StageRoot 'bootstrap') ` + -Destination $bootstrapRoot -Recurse + Copy-Item -LiteralPath $signerCertificatePath ` + -Destination (Join-Path $ResultsRoot 'publisher-contract.cer') + & (Join-Path $StageRoot 'tests\bootstrap-winget-sandbox.ps1') ` + -WorkRoot $ResultsRoot + + $harness = Join-Path $StageRoot 'tests\sandbox-conformance.ps1' + $feedServer = Join-Path $StageRoot 'tests\serve-feed.ps1' + $makeAppx = Join-Path $StageRoot 'tools\sdk-x64\makeappx.exe' + $signTool = Join-Path $StageRoot 'tools\sdk-x64\signtool.exe' + $publishers = @( + [pscustomobject]@{ + Name = 'Rust' + Cli = Join-Path $StageRoot 'tools\pinget-rust.exe' + Source = 'pinget-publisher-rust-conformance' + }, + [pscustomobject]@{ + Name = 'Dotnet' + Cli = Join-Path $StageRoot 'tools\dotnet\pinget.exe' + Source = 'pinget-publisher-dotnet-conformance' + } + ) + + foreach ($publisher in $publishers) { + $workRoot = Join-Path $ResultsRoot "work-$($publisher.Name)" + New-Item -ItemType Directory -Path $workRoot | Out-Null + $publisherInput = Join-Path $workRoot 'input' + New-Item -ItemType Directory -Path $publisherInput | Out-Null + Copy-Item -Path (Join-Path $StageRoot 'input\*') ` + -Destination $publisherInput -Recurse + Copy-Item -LiteralPath $feedServer -Destination (Join-Path $workRoot 'serve-feed.ps1') + Copy-Item -LiteralPath (Join-Path $ResultsRoot 'tls-server.pfx') ` + -Destination (Join-Path $workRoot 'tls-server.pfx') + Copy-Item -LiteralPath (Join-Path $ResultsRoot 'tls-server.cer') ` + -Destination (Join-Path $workRoot 'tls-server.cer') + + Write-Output "START signed stock WinGet acceptance: $($publisher.Name)" + & $harness ` + -WorkRoot $workRoot ` + -SourceName $publisher.Source ` + -ExecutionTarget Sandbox ` + -ExpectedWinGetVersion '1.29.380.0' ` + -SignerCertificatePath $signerCertificatePath ` + -PublisherCliPath $publisher.Cli ` + -PublisherStateDirectory (Join-Path $workRoot 'publisher-state') ` + -PublisherInputDirectory $publisherInput ` + -PfxPath $pfxPath ` + -PfxPasswordEnvironmentVariable 'PINGET_PUBLISHER_PFX_PASSWORD' ` + -RustReaderCliPath (Join-Path $StageRoot 'tools\pinget-rust.exe') ` + -DotnetReaderCliPath (Join-Path $StageRoot 'tools\dotnet\pinget.exe') ` + -PublisherName 'Pinget.Publisher.Conformance' ` + -PublisherIdentity 'CN=Pinget Publisher Conformance Test' ` + -InitialPackageVersion '1.0.0.0' ` + -MakeAppxPath $makeAppx ` + -SignToolPath $signTool + + $runSummary.Add([pscustomobject]@{ + publisher = $publisher.Name + result = 'passed' + sourceName = $publisher.Source + multiblockVersion = '1.0.499' + }) + } +} +catch { + $exitCode = 1 + Write-Warning $_ + $runSummary.Add([pscustomobject]@{ + publisher = 'gate' + result = 'failed' + error = $_.Exception.ToString() + }) +} +finally { + try { + $tlsCertificatePath = Join-Path $ResultsRoot 'tls-server.cer' + if (Test-Path -LiteralPath $tlsCertificatePath) { + $tlsCertificate = + [System.Security.Cryptography.X509Certificates.X509Certificate2]::new( + $tlsCertificatePath) + $testRootThumbprints.Add($tlsCertificate.Thumbprint) | Out-Null + $tlsCertificate.Dispose() + } + + foreach ($location in @( + [System.Security.Cryptography.X509Certificates.StoreLocation]::CurrentUser, + [System.Security.Cryptography.X509Certificates.StoreLocation]::LocalMachine + )) { + foreach ($storeName in @( + [System.Security.Cryptography.X509Certificates.StoreName]::TrustedPeople, + [System.Security.Cryptography.X509Certificates.StoreName]::Root + )) { + $store = [System.Security.Cryptography.X509Certificates.X509Store]::new( + $storeName, $location) + try { + $store.Open( + [System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite) + foreach ($certificate in @($store.Certificates)) { + if ($testRootThumbprints.Contains($certificate.Thumbprint)) { + $store.Remove($certificate) + $certificate.Dispose() + } + } + } + finally { + $store.Close() + } + } + } + } + catch { + $exitCode = 1 + $runSummary.Add([pscustomobject]@{ + publisher = 'cleanup' + result = 'failed' + error = $_.Exception.ToString() + }) + } + + if ($env:PINGET_PUBLISHER_PFX_PASSWORD) { + Remove-Item Env:\PINGET_PUBLISHER_PFX_PASSWORD -ErrorAction SilentlyContinue + } + try { + Stop-Transcript | Out-Null + } + catch { + } + [System.IO.File]::WriteAllText( + $summaryPath, + ($runSummary | ConvertTo-Json -Depth 6), + [System.Text.UTF8Encoding]::new($false)) +} + +if ($exitCode -ne 0) { + exit $exitCode +} diff --git a/tests/publisher/sandbox-conformance.ps1 b/tests/publisher/sandbox-conformance.ps1 new file mode 100644 index 0000000..4b56667 --- /dev/null +++ b/tests/publisher/sandbox-conformance.ps1 @@ -0,0 +1,719 @@ +param( + [string] $WorkRoot = 'C:\WinApp\work\PublisherContract', + [string] $SourceName = 'pinget-publisher-contract', + [string] $BaseUrl = 'https://127.0.0.1:8765/', + [switch] $ReuseExistingSource, + [string] $SignerCertificatePath, + [ValidateSet('Sandbox', 'EphemeralHost')] + [string] $ExecutionTarget = 'EphemeralHost', + [switch] $AllowEphemeralHostMutations, + [string] $ExpectedWinGetVersion = '1.29.380.0', + [string] $PublisherCliPath, + [string] $PublisherStateDirectory, + [string] $PublisherInputDirectory, + [string] $PfxPath, + [string] $PfxPasswordEnvironmentVariable, + [string] $RustReaderCliPath, + [string] $DotnetReaderCliPath, + [string] $PublisherName = 'Pinget.Publisher.Conformance', + [string] $PublisherIdentity = 'CN=Pinget Publisher Conformance Test', + [string] $InitialPackageVersion = '1.0.0.0', + [string] $MakeAppxPath, + [string] $SignToolPath +) + +$ErrorActionPreference = 'Stop' +if ($ExecutionTarget -eq 'EphemeralHost' -and -not $AllowEphemeralHostMutations) { + throw 'Host stock-source/certificate mutations are disabled. Use -ExecutionTarget Sandbox, or explicitly opt in with -AllowEphemeralHostMutations on a disposable CI runner.' +} +if ($ExecutionTarget -eq 'Sandbox' -and $AllowEphemeralHostMutations) { + throw '-AllowEphemeralHostMutations is valid only for an explicitly disposable host runner.' +} +$publisherArguments = @( + $PublisherCliPath, $PublisherStateDirectory, $PublisherInputDirectory, + $PfxPath, $PfxPasswordEnvironmentVariable, $MakeAppxPath, $SignToolPath +) +if ($publisherArguments.Where({ -not [string]::IsNullOrWhiteSpace($_) }).Count -gt 0 -and + $publisherArguments.Where({ [string]::IsNullOrWhiteSpace($_) }).Count -gt 0) { + throw 'Production publisher validation requires CLI, state, input, PFX, password environment variable, MakeAppx, and SignTool paths together.' +} +$readerArguments = @($RustReaderCliPath, $DotnetReaderCliPath) +if ($readerArguments.Where({ -not [string]::IsNullOrWhiteSpace($_) }).Count -notin @(0, 2)) { + throw 'Pinget reader conformance requires both the Rust and .NET CLI paths.' +} +$feed = Join-Path $WorkRoot 'feed' +$serverScript = Join-Path $WorkRoot 'serve-feed.ps1' +$serverCertificate = Join-Path $WorkRoot 'tls-server.pfx' +$downloadDirectory = Join-Path $WorkRoot ('downloaded-' + [Guid]::NewGuid().ToString('N')) +$packageId = 'Pinget.Contract.Singleton' +$addedPackageId = 'Pinget.Contract.Added' +$multiBlockPackageId = 'Pinget.Contract.MultiBlock' +$expectedPayload = 'Harmless Pinget contract fixture. Download only; never execute.' + "`n" +$expectedHash = '96c3a679ab1dc50990f84b7610f41bad666d92a82aceaf2edb8674a53587f5b7' +$script:addedTrustThumbprints = [System.Collections.Generic.List[string]]::new() +$script:sourceAdded = $false + +function Add-TestRootCertificate { + param([string] $Path) + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path)) { + return + } + $certificate = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new($Path) + $rootStore = [System.Security.Cryptography.X509Certificates.X509Store]::new( + [System.Security.Cryptography.X509Certificates.StoreName]::Root, + [System.Security.Cryptography.X509Certificates.StoreLocation]::LocalMachine + ) + try { + $rootStore.Open( + [System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite + ) + $alreadyTrusted = $rootStore.Certificates | + Where-Object { $_.Thumbprint -eq $certificate.Thumbprint } + if ($null -eq $alreadyTrusted) { + $rootStore.Add($certificate) + $script:addedTrustThumbprints.Add($certificate.Thumbprint) + } + } + finally { + $rootStore.Close() + $certificate.Dispose() + } +} + +function Remove-TestRootCertificates { + if ($script:addedTrustThumbprints.Count -eq 0) { + return + } + $rootStore = [System.Security.Cryptography.X509Certificates.X509Store]::new( + [System.Security.Cryptography.X509Certificates.StoreName]::Root, + [System.Security.Cryptography.X509Certificates.StoreLocation]::LocalMachine + ) + try { + $rootStore.Open( + [System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite + ) + foreach ($thumbprint in $script:addedTrustThumbprints) { + $matches = @($rootStore.Certificates | Where-Object { $_.Thumbprint -eq $thumbprint }) + foreach ($certificate in $matches) { + $rootStore.Remove($certificate) + $certificate.Dispose() + } + } + } + finally { + $rootStore.Close() + } +} + +function Invoke-PublisherCommand { + param([string[]] $Arguments) + $output = & $script:PublisherCliPath --output json repository @Arguments 2>&1 + $exitCode = $LASTEXITCODE + $text = ($output | ForEach-Object { $_.ToString() }) -join "`n" + if ($exitCode -ne 0) { + throw "Production publisher CLI exited with $exitCode for '$($Arguments -join ' ')':`n$text" + } + try { + return $text | ConvertFrom-Json + } + catch { + throw "Production publisher CLI did not return valid JSON for '$($Arguments -join ' ')':`n$text" + } +} + +function Publish-ProductionInput { + param([string] $InputDirectory, [string] $PackageVersion) + $build = Invoke-PublisherCommand @( + 'build', '--input', $InputDirectory, '--state', $script:PublisherStateDirectory + ) + if ([string]::IsNullOrWhiteSpace($build.buildId)) { + throw 'Production publisher build returned no buildId.' + } + $published = Invoke-PublisherCommand @( + 'publish', '--state', $script:PublisherStateDirectory, + '--destination', $script:feed, '--name', $script:PublisherName, + '--publisher', $script:PublisherIdentity, '--version', $PackageVersion, + '--pfx', $script:PfxPath, '--password-env', $script:PfxPasswordEnvironmentVariable, + '--makeappx', $script:MakeAppxPath, '--signtool', $script:SignToolPath, + '--build', $build.buildId + ) + if (-not $published.valid) { + throw "Production publisher did not verify its signed output: $($published.message)" + } + return $build.buildId +} + +function Invoke-PingetReaderCommand { + param( + [string] $CliPath, + [string] $AppRoot, + [string[]] $Arguments + ) + $previousAppRoot = [Environment]::GetEnvironmentVariable('PINGET_APPROOT', 'Process') + $previousSourceMode = [Environment]::GetEnvironmentVariable('PINGET_SOURCE_MODE', 'Process') + try { + [Environment]::SetEnvironmentVariable('PINGET_APPROOT', $AppRoot, 'Process') + [Environment]::SetEnvironmentVariable('PINGET_SOURCE_MODE', 'private', 'Process') + $output = & $CliPath @Arguments 2>&1 + $exitCode = $LASTEXITCODE + $text = ($output | ForEach-Object { $_.ToString() }) -join "`n" + if ($exitCode -ne 0) { + throw "Pinget reader CLI exited with $exitCode for '$($Arguments -join ' ')':`n$text" + } + return $text + } + finally { + [Environment]::SetEnvironmentVariable('PINGET_APPROOT', $previousAppRoot, 'Process') + [Environment]::SetEnvironmentVariable('PINGET_SOURCE_MODE', $previousSourceMode, 'Process') + } +} + +function Test_PingetReader_SearchAndShowReadsMultiBlockVersionData { + if ($script:ReaderCliPaths.Count -eq 0) { + return + } + + $winGetSourcesBefore = (Invoke-WinGetChecked @( + 'source', 'list', '--disable-interactivity' + )).Trim() + foreach ($reader in $script:ReaderCliPaths) { + $appRoot = Join-Path $script:WorkRoot "pinget-reader-$($reader.Name.ToLowerInvariant())" + if (Test-Path -LiteralPath $appRoot) { + throw "Refusing to overwrite app-private Pinget source root: $appRoot" + } + $readerSourceName = "pinget-private-reader-$($reader.Name.ToLowerInvariant())" + $sourceAdded = $false + try { + $null = Invoke-PingetReaderCommand $reader.Path $appRoot @( + 'source', 'add', '--name', $readerSourceName, + '--arg', $script:BaseUrl, '--type', 'Microsoft.PreIndexed.Package', + '--trust-level', 'Trusted' + ) + $sourceAdded = $true + $null = Invoke-PingetReaderCommand $reader.Path $appRoot @( + 'source', 'update', $readerSourceName + ) + $search = Invoke-PingetReaderCommand $reader.Path $appRoot @( + 'search', '--id', $script:multiBlockPackageId, + '--source', $readerSourceName, '--exact' + ) + if ($search -notmatch [regex]::Escape($script:multiBlockPackageId) -or + $search -notmatch '(?m)\b1\.0\.499\b') { + throw "$($reader.Name) search did not return the final multi-block version 1.0.499." + } + + $show = Invoke-PingetReaderCommand $reader.Path $appRoot @( + 'show', '--id', $script:multiBlockPackageId, + '--source', $readerSourceName, '--exact' + ) + if ($show -notmatch [regex]::Escape($script:multiBlockPackageId) -or + $show -notmatch '(?m)^Version:\s*1\.0\.499\s*$') { + throw "$($reader.Name) show did not read the final version 1.0.499." + } + + $versions = Invoke-PingetReaderCommand $reader.Path $appRoot @( + 'show', '--id', $script:multiBlockPackageId, + '--source', $readerSourceName, '--exact', '--versions' + ) + $versionLines = @( + $versions -split '\r?\n' | + ForEach-Object { $_.Trim() } | + Where-Object { $_ -match '^1\.0\.\d{1,3}$' } + ) + if ($versionLines.Count -ne 500 -or $versionLines[0] -ne '1.0.499') { + throw "$($reader.Name) show --versions did not return all 500 descending versions with 1.0.499 first." + } + } + finally { + if ($sourceAdded) { + $null = Invoke-PingetReaderCommand $reader.Path $appRoot @( + 'source', 'remove', $readerSourceName + ) + } + } + + $winGetSourcesAfter = (Invoke-WinGetChecked @( + 'source', 'list', '--disable-interactivity' + )).Trim() + if ($winGetSourcesAfter -ne $winGetSourcesBefore) { + throw "$($reader.Name) private source operations changed the stock WinGet source list." + } + Write-Output "PASS Test_Pinget_$($reader.Name)_SearchAndShowReadsMultiBlockVersionData" + } +} + +function New-AddedPackageManifest { + return @' +PackageIdentifier: Pinget.Contract.Added +PackageVersion: 1.0.0 +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PublisherUrl: https://example.invalid/ +PackageName: Pinget Added Fixture +License: MIT +ShortDescription: Harmless added publisher conformance package +Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://127.0.0.1:8765/HarmlessPayload.txt + InstallerSha256: 96c3a679ab1dc50990f84b7610f41bad666d92a82aceaf2edb8674a53587f5b7 + Scope: user +ManifestType: singleton +ManifestVersion: 1.9.0 +'@ +} + +function Initialize-MultiBlockPublisherInput { + $directory = Join-Path $script:PublisherInputDirectory 'multiblock' + if (Test-Path -LiteralPath $directory) { + $existing = @(Get-ChildItem -LiteralPath $directory -Filter '*.yaml' -File) + if ($existing.Count -ne 500) { + throw "Expected the existing multi-block fixture to contain exactly 500 manifests; found $($existing.Count)." + } + return + } + + New-Item -ItemType Directory -Path $directory | Out-Null + for ($i = 0; $i -lt 500; $i++) { + $version = "1.0.$i" + $manifest = @" +PackageIdentifier: $script:multiBlockPackageId +PackageVersion: $version +PackageLocale: en-US +Publisher: Pinget Test Fixtures +PublisherUrl: https://example.invalid/ +PackageName: Pinget Multi-Block Fixture +License: MIT +ShortDescription: Large harmless version-data fixture +Installers: + - Architecture: x64 + InstallerType: exe + InstallerUrl: https://example.invalid/Pinget.MultiBlock/$version.exe + InstallerSha256: $([string]::new('a', 64)) + Scope: user +ManifestType: singleton +ManifestVersion: 1.9.0 +"@ + $path = Join-Path $directory ("version-{0:D3}.yaml" -f $i) + [System.IO.File]::WriteAllText( + $path, + $manifest + "`n", + [System.Text.UTF8Encoding]::new($false)) + } +} + +function Invoke-UpdateSource { + $null = Invoke-WinGetChecked @( + 'source', 'update', '--name', $script:SourceName, '--disable-interactivity' + ) +} + +function Invoke-WinGetRaw { + param([string[]] $Arguments) + $output = & $script:winget @Arguments 2>&1 + return [pscustomobject]@{ + ExitCode = $LASTEXITCODE + Text = ($output | ForEach-Object { $_.ToString() }) -join "`n" + } +} + +function Invoke-WinGetChecked { + param([string[]] $Arguments) + Write-Host ('COMMAND: winget ' + ($Arguments -join ' ')) + $output = & $script:winget @Arguments 2>&1 + $exitCode = $LASTEXITCODE + $output | ForEach-Object { Write-Host $_ } + if ($exitCode -ne 0) { + throw "winget exited with $exitCode for: $($Arguments -join ' ')" + } + return ($output -join "`n") +} + +function Test_StockWinGet_SourceAddAndUpdate { + if ($ReuseExistingSource) { + $existing = Invoke-WinGetChecked @( + 'source', 'list', '--name', $script:SourceName, + '--disable-interactivity' + ) + $matchesExpectedSource = + $existing -match ('(?m)^\s*Name\s+' + [regex]::Escape($script:SourceName) + '\s*$') -and + $existing -match '(?m)^\s*Type\s+Microsoft\.PreIndexed\.Package\s*$' -and + $existing -match ('(?m)^\s*Argument\s+' + [regex]::Escape($script:BaseUrl) + '\s*$') -and + $existing -match '(?m)^\s*Trust Level\s+Trusted\s*$' + if (-not $matchesExpectedSource) { + throw "Existing WinGet source '$($script:SourceName)' does not match the requested production feed configuration." + } + Write-Output 'PASS Test_StockWinGet_SourceReuseMatchesProductionFeed' + } + else { + $null = Invoke-WinGetChecked @( + 'source', 'add', '--name', $script:SourceName, + '--arg', $script:BaseUrl, '--type', 'Microsoft.PreIndexed.Package', + '--trust-level', 'trusted', '--accept-source-agreements' + ) + $script:sourceAdded = $true + Write-Output 'PASS Test_StockWinGet_SourceAdd' + } + $null = Invoke-WinGetChecked @( + 'source', 'update', '--name', $script:SourceName, + '--disable-interactivity' + ) + Write-Output 'PASS Test_StockWinGet_SourceUpdate' +} + +function Test_StockWinGet_SearchFindsFixture { + $output = Invoke-WinGetChecked @( + 'search', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($output -notmatch [regex]::Escape($script:packageId)) { + throw 'Search output did not contain the exact fixture package identifier.' + } + Write-Output 'PASS Test_StockWinGet_SearchFindsFixture' +} + +function Test_StockWinGet_ShowReadsFixtureManifest { + $output = Invoke-WinGetChecked @( + 'show', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($output -notmatch 'Pinget Contract Fixture' -or + $output -notmatch '1\.0\.0') { + throw 'Show output did not expose the expected name and version.' + } + Write-Output 'PASS Test_StockWinGet_ShowReadsFixtureManifest' +} + +function Test_StockWinGet_ShowReadsMultiBlockVersionData { + $output = Invoke-WinGetChecked @( + 'show', '--id', $script:multiBlockPackageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($output -notmatch [regex]::Escape($script:multiBlockPackageId) -or + $output -notmatch [regex]::Escape('1.0.499') -or + $output -notmatch 'Pinget Multi-Block Fixture') { + throw 'Show output did not expose the last version from the multi-block version-data sidecar.' + } + Write-Output 'PASS Test_StockWinGet_ShowReadsMultiBlockVersionData' +} + +function Test_StockWinGet_DownloadVerifiesHarmlessPayload { + if (Test-Path $script:downloadDirectory) { + throw "Refusing to overwrite download evidence at $script:downloadDirectory" + } + New-Item -ItemType Directory -Path $script:downloadDirectory | Out-Null + $null = Invoke-WinGetChecked @( + 'download', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--download-directory', + $script:downloadDirectory, '--accept-source-agreements', + '--accept-package-agreements', '--disable-interactivity' + ) + $payload = @( + Get-ChildItem -LiteralPath $script:downloadDirectory -File | + Where-Object { + (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant() -eq + $script:expectedHash + } + ) + if ($payload.Count -ne 1) { + throw "Expected one downloaded file with the fixture payload hash; found $($payload.Count)." + } + $actualHash = (Get-FileHash -LiteralPath $payload.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actualHash -ne $script:expectedHash) { + throw "Downloaded payload hash mismatch: $actualHash" + } + $actualText = [System.IO.File]::ReadAllText($payload.FullName) + if ($actualText -ne $script:expectedPayload) { + throw 'Downloaded payload bytes did not match the locally authored fixture.' + } + Write-Output 'PASS Test_StockWinGet_DownloadVerifiesHarmlessPayload' +} + +function Test_StockWinGet_SourceVersionBumpAddRemove { + if ([string]::IsNullOrWhiteSpace($script:PublisherCliPath)) { + return + } + + $bumpInput = Join-Path $script:WorkRoot 'version-bump-input' + $removeInput = Join-Path $script:WorkRoot 'remove-package-input' + foreach ($path in @($bumpInput, $removeInput)) { + if (Test-Path -LiteralPath $path) { + throw "Refusing to overwrite publisher update fixture directory: $path" + } + } + Copy-Item -LiteralPath $script:PublisherInputDirectory -Destination $bumpInput -Recurse + $baseManifest = Join-Path $bumpInput 'Pinget.Contract.Singleton.yaml' + $manifestText = [System.IO.File]::ReadAllText($baseManifest) + $manifestText = [regex]::Replace($manifestText, '(?m)^PackageVersion: 1\.0\.0\r?$', 'PackageVersion: 1.1.0') + $manifestText = [regex]::Replace($manifestText, '(?m)^PackageName: Pinget Contract Fixture\r?$', 'PackageName: Pinget Contract Fixture Updated') + $manifestText = [regex]::Replace($manifestText, '(?m)^ShortDescription: .*$', 'ShortDescription: Updated harmless production publisher conformance fixture') + [System.IO.File]::WriteAllText( + $baseManifest, + $manifestText, + [System.Text.UTF8Encoding]::new($false)) + [System.IO.File]::WriteAllText( + (Join-Path $bumpInput 'Pinget.Contract.Added.yaml'), + (New-AddedPackageManifest), + [System.Text.UTF8Encoding]::new($false)) + + $null = Publish-ProductionInput $bumpInput '1.0.0.1' + $cached = Invoke-WinGetChecked @( + 'show', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($cached -notmatch '1\.0\.0' -or + $cached -notmatch 'Pinget Contract Fixture' -or + $cached -match '1\.1\.0') { + throw 'Cached source metadata changed before an explicit source update during publication.' + } + Write-Output 'PASS Test_StockWinGet_CachedPreviousIndexRetainsMetadataDuringPublish' + + $null = Invoke-WinGetChecked @( + 'source', 'update', '--name', $script:SourceName, '--disable-interactivity' + ) + $updated = Invoke-WinGetChecked @( + 'show', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($updated -notmatch '1\.1\.0' -or + $updated -notmatch 'Pinget Contract Fixture Updated') { + throw 'Updated source did not expose the bumped package version and metadata.' + } + $added = Invoke-WinGetChecked @( + 'search', '--id', $script:addedPackageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($added -notmatch [regex]::Escape($script:addedPackageId)) { + throw 'Source update did not expose the newly added package.' + } + Write-Output 'PASS Test_StockWinGet_SourceVersionBumpAndPackageAddition' + + Copy-Item -LiteralPath $bumpInput -Destination $removeInput -Recurse + Remove-Item -LiteralPath (Join-Path $removeInput 'Pinget.Contract.Singleton.yaml') + $null = Publish-ProductionInput $removeInput '1.0.0.2' + $cachedBeforeRemoval = Invoke-WinGetChecked @( + 'show', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($cachedBeforeRemoval -notmatch '1\.1\.0' -or + $cachedBeforeRemoval -notmatch 'Pinget Contract Fixture Updated') { + throw 'Cached prior-index metadata was not available while the removal publication was staged.' + } + + $null = Invoke-WinGetChecked @( + 'source', 'update', '--name', $script:SourceName, '--disable-interactivity' + ) + $removed = Invoke-WinGetRaw @( + 'search', '--id', $script:packageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($removed.Text -match [regex]::Escape($script:packageId)) { + throw 'Removed package remained searchable after the source update.' + } + $stillPresent = Invoke-WinGetChecked @( + 'search', '--id', $script:addedPackageId, '--exact', + '--source', $script:SourceName, '--accept-source-agreements', + '--disable-interactivity' + ) + if ($stillPresent -notmatch [regex]::Escape($script:addedPackageId)) { + throw 'Unremoved package disappeared from the updated source.' + } + + $marker = Get-Content -LiteralPath (Join-Path $script:feed '.pinget-publisher.json') -Raw | + ConvertFrom-Json + $retainedOldManifest = $marker.artifacts.PSObject.Properties.Name | + Where-Object { $_ -like 'manifests/Pinget.Contract.Singleton/1.0.0/*' } | + Select-Object -First 1 + if ([string]::IsNullOrWhiteSpace($retainedOldManifest) -or + -not (Test-Path -LiteralPath (Join-Path $script:feed $retainedOldManifest.Replace('/', '\')))) { + throw 'The removal publication did not retain the previously published immutable manifest.' + } + $verified = Invoke-PublisherCommand @( + 'verify', '--destination', $script:feed, '--signtool', $script:SignToolPath + ) + if (-not $verified.valid) { + throw 'Published destination did not verify after add/remove updates.' + } + Write-Output 'PASS Test_StockWinGet_SourceRemovalDropsPackageAndRetainsPublishedArtifacts' +} + +$script:WorkRoot = $WorkRoot +$script:feed = $feed +$script:PublisherCliPath = $PublisherCliPath +$script:PublisherStateDirectory = $PublisherStateDirectory +$script:PublisherInputDirectory = $PublisherInputDirectory +$script:PfxPath = $PfxPath +$script:PfxPasswordEnvironmentVariable = $PfxPasswordEnvironmentVariable +$script:ReaderCliPaths = @( + if (-not [string]::IsNullOrWhiteSpace($RustReaderCliPath)) { + [pscustomobject]@{ Name = 'Rust'; Path = $RustReaderCliPath } + } + if (-not [string]::IsNullOrWhiteSpace($DotnetReaderCliPath)) { + [pscustomobject]@{ Name = 'Dotnet'; Path = $DotnetReaderCliPath } + } +) +$script:PublisherName = $PublisherName +$script:PublisherIdentity = $PublisherIdentity +$script:MakeAppxPath = $MakeAppxPath +$script:SignToolPath = $SignToolPath +$script:SourceName = $SourceName +$script:BaseUrl = $BaseUrl +$script:packageId = $packageId +$script:addedPackageId = $addedPackageId +$script:multiBlockPackageId = $multiBlockPackageId +$server = $null + +try { + $publisherArguments = @( + $script:PublisherCliPath, $script:PublisherStateDirectory, $script:PublisherInputDirectory, + $script:PfxPath, $script:PfxPasswordEnvironmentVariable, $script:MakeAppxPath, $script:SignToolPath + ) + $presentCount = @($publisherArguments | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }).Count + if ($presentCount -ne 0 -and $presentCount -ne $publisherArguments.Count) { + throw 'Production publisher validation requires CLI, state, input, PFX, password environment variable, MakeAppx, and SignTool paths together.' + } + + foreach ($requiredPath in @($serverScript, $serverCertificate)) { + if (-not (Test-Path -LiteralPath $requiredPath)) { + throw "Required stock WinGet fixture server input is missing: $requiredPath" + } + } + foreach ($readerPath in @($RustReaderCliPath, $DotnetReaderCliPath) | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) { + if (-not (Test-Path -LiteralPath $readerPath)) { + throw "Required Pinget reader CLI does not exist: $readerPath" + } + } + if (-not [string]::IsNullOrWhiteSpace($script:PublisherCliPath)) { + if ([string]::IsNullOrWhiteSpace($SignerCertificatePath)) { + throw 'Signed production publishing requires -SignerCertificatePath so the test certificate can be trusted only on the selected test machine.' + } + if (-not (Test-Path -LiteralPath $script:PublisherInputDirectory)) { + throw "Production publisher input is missing: $script:PublisherInputDirectory" + } + if (-not (Test-Path -LiteralPath $script:PfxPath)) { + throw "Production publisher PFX is missing: $script:PfxPath" + } + if ([string]::IsNullOrEmpty([Environment]::GetEnvironmentVariable($script:PfxPasswordEnvironmentVariable))) { + throw "The PFX password environment variable '$script:PfxPasswordEnvironmentVariable' is not set." + } + if (-not (Test-Path -LiteralPath $script:MakeAppxPath) -or + -not (Test-Path -LiteralPath $script:SignToolPath)) { + throw 'The supplied MakeAppx or SignTool path does not exist on the selected test machine.' + } + if (Test-Path -LiteralPath (Join-Path $feed 'source2.msix')) { + throw "Refusing to publish over an existing feed: $feed" + } + New-Item -ItemType Directory -Force -Path $feed | Out-Null + } + elseif (-not (Test-Path -LiteralPath (Join-Path $feed 'source2.msix'))) { + throw "Fixture source2.msix not found under $feed" + } + + if (-not [string]::IsNullOrWhiteSpace($SignerCertificatePath)) { + Add-TestRootCertificate $SignerCertificatePath + Write-Output 'Trusted the ephemeral production signer only in the selected test machine Root store.' + } + Add-TestRootCertificate (Join-Path $WorkRoot 'tls-server.cer') + + if (-not [string]::IsNullOrWhiteSpace($script:PublisherCliPath)) { + Initialize-MultiBlockPublisherInput + $null = Publish-ProductionInput $script:PublisherInputDirectory $InitialPackageVersion + } + + $appInstaller = Get-AppxPackage -Name Microsoft.DesktopAppInstaller + if ($null -eq $appInstaller -or $appInstaller.Version.ToString() -ne $ExpectedWinGetVersion) { + $actualVersion = if ($null -eq $appInstaller) { '' } else { $appInstaller.Version.ToString() } + throw "Expected stock WinGet $ExpectedWinGetVersion, found $actualVersion" + } + $script:winget = Join-Path $env:LOCALAPPDATA 'Microsoft\WindowsApps\winget.exe' + if (-not (Test-Path $script:winget)) { + $command = Get-Command winget.exe -ErrorAction SilentlyContinue + if ($null -eq $command) { + throw 'WinGet package registered but winget.exe execution alias is unavailable.' + } + $script:winget = $command.Source + } + $versionOutput = & $script:winget --version 2>&1 + $expectedClientVersion = $ExpectedWinGetVersion -replace '\.0$', '' + if ($LASTEXITCODE -ne 0 -or ($versionOutput -join '') -notmatch [regex]::Escape($expectedClientVersion)) { + throw "Unexpected WinGet client version: $($versionOutput -join ' ')" + } + Write-Output "Stock WinGet: $($versionOutput -join ' ')" + + $serverStdout = Join-Path $WorkRoot 'server.stdout.txt' + $serverStderr = Join-Path $WorkRoot 'server.stderr.txt' + $harmlessPayloadPath = Join-Path $WorkRoot 'HarmlessPayload.txt' + [System.IO.File]::WriteAllText( + $harmlessPayloadPath, + $expectedPayload, + [System.Text.UTF8Encoding]::new($false) + ) + $server = Start-Process -FilePath "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" ` + -ArgumentList @('-NoLogo', '-NoProfile', '-ExecutionPolicy', 'Bypass', + '-File', $serverScript, '-Root', $feed, '-CertificatePath', + $serverCertificate, '-Port', '8765', '-HarmlessPayloadPath', + $harmlessPayloadPath) ` + -PassThru -WindowStyle Hidden ` + -RedirectStandardOutput $serverStdout -RedirectStandardError $serverStderr + $ready = $false + for ($attempt = 0; $attempt -lt 20; $attempt++) { + try { + $response = Invoke-WebRequest -Uri ($script:BaseUrl + 'source2.msix') ` + -UseBasicParsing -TimeoutSec 2 + if ($response.StatusCode -eq 200) { + $ready = $true + break + } + } + catch { + Start-Sleep -Milliseconds 250 + } + } + if (-not $ready) { + $serverError = if (Test-Path $serverStderr) { + Get-Content -LiteralPath $serverStderr -Raw + } + else { + '' + } + throw "Sandbox-local fixture server did not become ready. Server stderr: $serverError" + } + Write-Output 'PASS Test_StockWinGet_FeedServedOnSandboxLoopback' + Test_PingetReader_SearchAndShowReadsMultiBlockVersionData + Test_StockWinGet_SourceAddAndUpdate + Test_StockWinGet_SearchFindsFixture + Test_StockWinGet_ShowReadsFixtureManifest + if (-not [string]::IsNullOrWhiteSpace($script:PublisherCliPath)) { + Test_StockWinGet_ShowReadsMultiBlockVersionData + } + Test_StockWinGet_DownloadVerifiesHarmlessPayload + Test_StockWinGet_SourceVersionBumpAddRemove +} +finally { + if ($null -ne $server -and -not $server.HasExited) { + Stop-Process -Id $server.Id + } + if ($script:sourceAdded) { + try { + & $script:winget source remove --name $SourceName --disable-interactivity 2>&1 | Out-Null + } + catch { + Write-Warning "Could not remove temporary WinGet source '$SourceName': $($_.Exception.Message)" + } + } + try { + Remove-TestRootCertificates + } + catch { + Write-Warning "Could not remove test-added certificate trust: $($_.Exception.Message)" + } +} diff --git a/tests/publisher/serve-feed.ps1 b/tests/publisher/serve-feed.ps1 new file mode 100644 index 0000000..56e9a07 --- /dev/null +++ b/tests/publisher/serve-feed.ps1 @@ -0,0 +1,136 @@ +param( + [Parameter(Mandatory = $true)] + [string] $Root, + [Parameter(Mandatory = $true)] + [string] $CertificatePath, + [int] $Port = 8765, + [string] $HarmlessPayloadPath +) + +$ErrorActionPreference = 'Stop' +$rootPath = [System.IO.Path]::GetFullPath($Root).TrimEnd('\') + '\' +$listener = [System.Net.Sockets.TcpListener]::new( + [System.Net.IPAddress]::Loopback, + $Port +) +$certificate = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new( + $CertificatePath, + [string]::Empty, + [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::Exportable ` + -bor [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::UserKeySet +) +$listener.Start() +Write-Output "Serving $rootPath over TLS on 127.0.0.1:$Port" + +try { + while ($true) { + $client = $listener.AcceptTcpClient() + $tls = $null + try { + $tls = [System.Net.Security.SslStream]::new( + $client.GetStream(), + $false + ) + $tls.AuthenticateAsServer( + $certificate, + $false, + [System.Security.Authentication.SslProtocols]::Tls12, + $false + ) + $requestBuffer = [System.IO.MemoryStream]::new() + try { + $previous = 0 + $current = 0 + while (($current = $tls.ReadByte()) -ge 0) { + $requestBuffer.WriteByte([byte]$current) + if ($previous -eq 13 -and $current -eq 10) { + $bytes = $requestBuffer.ToArray() + $length = $bytes.Length + if ($length -ge 4 -and + $bytes[$length - 4] -eq 13 -and + $bytes[$length - 3] -eq 10 -and + $bytes[$length - 2] -eq 13 -and + $bytes[$length - 1] -eq 10) { + break + } + } + $previous = $current + } + $requestText = [System.Text.Encoding]::ASCII.GetString( + $requestBuffer.ToArray() + ) + } + finally { + $requestBuffer.Dispose() + } + + $requestLine = ($requestText -split "`r`n", 2)[0] + $parts = $requestLine -split ' ', 3 + if ($parts.Length -lt 2 -or $parts[0] -notin @('GET', 'HEAD')) { + $status = '405 Method Not Allowed' + $body = [byte[]]::new(0) + $contentLength = 0 + } + else { + $relative = [System.Uri]::UnescapeDataString( + ([System.Uri]::new("http://127.0.0.1:$Port$($parts[1])")).AbsolutePath.TrimStart('/') + ).Replace('/', '\') + if ([string]::IsNullOrWhiteSpace($relative)) { + $relative = 'source2.msix' + } + $isHarmlessPayload = $relative.Equals( + 'HarmlessPayload.txt', + [System.StringComparison]::OrdinalIgnoreCase + ) -and -not [string]::IsNullOrWhiteSpace($HarmlessPayloadPath) + if ($isHarmlessPayload) { + $filePath = [System.IO.Path]::GetFullPath($HarmlessPayloadPath) + $allowedPath = $true + } + else { + $filePath = [System.IO.Path]::GetFullPath( + [System.IO.Path]::Combine($rootPath, $relative) + ) + $allowedPath = $filePath.StartsWith( + $rootPath, + [System.StringComparison]::OrdinalIgnoreCase + ) + } + if (-not $allowedPath -or -not [System.IO.File]::Exists($filePath)) { + $status = '404 Not Found' + $body = [byte[]]::new(0) + $contentLength = 0 + } + else { + $status = '200 OK' + $body = [System.IO.File]::ReadAllBytes($filePath) + $contentLength = $body.Length + if ($parts[0] -eq 'HEAD') { + $body = [byte[]]::new(0) + } + } + } + + $header = [System.Text.Encoding]::ASCII.GetBytes( + "HTTP/1.1 $status`r`n" + + "Content-Length: $contentLength`r`n" + + "Content-Type: application/octet-stream`r`n" + + "Connection: close`r`n`r`n" + ) + $tls.Write($header, 0, $header.Length) + if ($body.Length -gt 0) { + $tls.Write($body, 0, $body.Length) + } + $tls.Flush() + } + finally { + if ($null -ne $tls) { + $tls.Dispose() + } + $client.Dispose() + } + } +} +finally { + $listener.Stop() + $certificate.Dispose() +} diff --git a/tests/publisher/validate_production_snapshot.py b/tests/publisher/validate_production_snapshot.py new file mode 100644 index 0000000..ce60dcf --- /dev/null +++ b/tests/publisher/validate_production_snapshot.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Independently validate a Pinget build snapshot with sqlite3 and native MSZIP.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sqlite3 +import sys +from pathlib import Path + +import yaml + +from build_v2_fixture import decompress_mszip + + +EXPECTED_TABLES = { + "metadata", + "packages", + "tags2", + "tags2_map", + "commands2", + "commands2_map", + "pfns2", + "productcodes2", + "norm_names2", + "norm_publishers2", + "upgradecodes2", +} + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def contained_file(root: Path, relative: str) -> Path: + root = root.resolve(strict=True) + path = (root / Path(relative.replace("/", "\\"))).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise AssertionError(f"artifact path is not a contained file: {relative}") + return path + + +def version_records(text: str) -> list[dict[str, object]]: + document = yaml.safe_load(text) + if not isinstance(document, dict) or document.get("sV") != "1.0": + raise AssertionError("version sidecar has an invalid document or schema marker") + records = document.get("vD") + if not isinstance(records, list) or not records: + raise AssertionError("version sidecar has no vD records") + for record in records: + if not isinstance(record, dict): + raise AssertionError("version sidecar contains a non-object vD record") + if not all(isinstance(record.get(key), str) for key in ("v", "rP", "s256H")): + raise AssertionError("version sidecar record lacks string v, rP, or s256H fields") + if not re.fullmatch(r"[0-9a-f]{64}", record["s256H"]): + raise AssertionError("version sidecar contains an invalid lowercase SHA-256") + return records + + +def validate(snapshot: Path, expected_package: str | None) -> dict[str, object]: + snapshot = snapshot.resolve(strict=True) + metadata_path = snapshot / "snapshot.json" + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if metadata.get("schemaVersion") != 1: + raise AssertionError("snapshot schemaVersion is not 1") + if metadata.get("artifactFormatVersion") != 2: + raise AssertionError("snapshot artifactFormatVersion is not 2") + if not metadata.get("buildId"): + raise AssertionError("snapshot buildId is missing") + + feed = snapshot / "feed" + artifact_inventory = metadata.get("artifacts") + if not isinstance(artifact_inventory, dict): + raise AssertionError("snapshot artifacts must be a JSON object") + for relative, expected_hash in artifact_inventory.items(): + path = contained_file(feed, relative) + if digest(path) != expected_hash: + raise AssertionError(f"artifact hash mismatch: {relative}") + + index = snapshot / "Public" / "index.db" + with sqlite3.connect(f"file:{index.as_posix()}?mode=ro", uri=True) as db: + integrity = db.execute("PRAGMA integrity_check").fetchone()[0] + if integrity != "ok": + raise AssertionError(f"SQLite integrity_check returned {integrity!r}") + tables = { + row[0] + for row in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + } + if tables != EXPECTED_TABLES: + raise AssertionError(f"unexpected WinGet v2 table set: {sorted(tables)}") + metadata_rows = dict(db.execute("SELECT name,value FROM metadata")) + if (metadata_rows.get("majorVersion"), metadata_rows.get("minorVersion")) != ("2", "0"): + raise AssertionError("WinGet schema markers are not 2.0") + packages = db.execute( + "SELECT id,name,latest_version,hash,arp_min_version,arp_max_version " + "FROM packages ORDER BY id COLLATE NOCASE" + ).fetchall() + if not packages: + raise AssertionError("the production snapshot contains no packages") + if expected_package is not None and expected_package not in {row[0] for row in packages}: + raise AssertionError(f"expected package {expected_package!r} is absent") + + decoded_versions: dict[str, list[str]] = {} + for package_id, _name, latest_version, package_hash, _, _ in packages: + if package_hash is None or len(package_hash) != 32: + raise AssertionError(f"package {package_id} has no 32-byte sidecar hash") + full_hash = package_hash.hex() + sidecar_relative = ( + f"packages/{package_id}/{full_hash[:8]}/versionData.mszyml" + ) + sidecar_path = contained_file(feed, sidecar_relative) + if digest(sidecar_path) != full_hash: + raise AssertionError(f"package {package_id} sidecar hash/path is invalid") + raw_version_data = decompress_mszip(sidecar_path.read_bytes()) + version_text = raw_version_data.decode("utf-8") + if not version_text.startswith('sV: "1.0"\n'): + raise AssertionError(f"package {package_id} has an unexpected sidecar header") + records = version_records(version_text) + record_versions = [record["v"] for record in records] + if len(record_versions) != len(set(record_versions)): + raise AssertionError(f"package {package_id} has duplicate version records") + if record_versions[0] != latest_version: + raise AssertionError( + f"package {package_id} sidecar does not place latest version first" + ) + for record in records: + version = record["v"] + manifest_relative = record["rP"] + manifest_hash = record["s256H"] + manifest_path = contained_file(feed, manifest_relative) + if digest(manifest_path) != manifest_hash: + raise AssertionError( + f"package {package_id} manifest hash mismatch for {version}" + ) + if latest_version not in {record["v"] for record in records}: + raise AssertionError(f"latest version missing from sidecar for {package_id}") + arp_minimum, arp_maximum = next( + row[4:6] for row in packages if row[0] == package_id + ) + sidecar_minima = { + record.get("aMiV") + for record in records + if isinstance(record.get("aMiV"), str) + } + sidecar_maxima = { + record.get("aMaV") + for record in records + if isinstance(record.get("aMaV"), str) + } + if arp_minimum is not None and arp_minimum not in sidecar_minima: + raise AssertionError(f"minimum ARP version missing from sidecar for {package_id}") + if arp_maximum is not None and arp_maximum not in sidecar_maxima: + raise AssertionError(f"maximum ARP version missing from sidecar for {package_id}") + if package_id == "Pinget.Test.Matrix": + expected_arp = { + "1.9.0": ("1.9.0", "1.9.0"), + "1.10.0": ("1.10.0", "1.10.0"), + } + actual_arp = { + record["v"]: (record.get("aMiV"), record.get("aMaV")) + for record in records + } + if actual_arp != expected_arp or (arp_minimum, arp_maximum) != ( + "1.9.0", + "1.10.0", + ): + raise AssertionError( + f"per-version ARP metadata or package bounds are incorrect for {package_id}" + ) + decoded_versions[package_id] = record_versions + + return { + "snapshot": str(snapshot), + "buildId": metadata["buildId"], + "packages": {row[0]: row[2] for row in packages}, + "versions": decoded_versions, + "artifacts": len(artifact_inventory), + "sqlite": "ok", + "mszip": "native Windows Compression API", + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--snapshot", type=Path, required=True) + parser.add_argument("--expect-package") + args = parser.parse_args() + try: + print(json.dumps(validate(args.snapshot, args.expect_package), sort_keys=True)) + except Exception as error: + print(f"snapshot validation failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6b67555cfe8dc4516bce7ca383749e4e6ac86a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sat, 31 Oct 2026 14:14:47 -0400 Subject: [PATCH 2/8] Cover initial .NET publication cleanup and retry after metadata copies Use a controlled MSIX identity mismatch to exercise post-copy rollback, assert an empty destination and unchanged publication state, then verify a successful retry. Document remaining Rust and signed-client acceptance limits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/repository-publishing.md | 7 +- .../PublisherPackagingTests.cs | 78 +++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/docs/repository-publishing.md b/docs/repository-publishing.md index adb73ea..e295a2e 100644 --- a/docs/repository-publishing.md +++ b/docs/repository-publishing.md @@ -193,8 +193,11 @@ the local exact-byte comparison and independent SQLite/MSZIP checks, but have not received a fresh signed stock-client acceptance run. Failure tests cover packaging/signing errors, update promotion rollback and -destination locking. Initial publication failure after metadata copying, -followed by cleanup and retry, remains an end-to-end coverage gap. +destination locking. The .NET publisher also has an end-to-end regression +for initial publication failure after metadata copying: a controlled MSIX +identity mismatch leaves the destination empty and no publication recorded, +then a valid retry succeeds. The equivalent Rust post-copy failure path +remains unverified; Rust verifies the staged MSIX before copying metadata. Stock-client acceptance and signature trust checks belong in a fresh Windows Sandbox or an ephemeral Windows CI runner. They must not register test sources diff --git a/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs b/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs index 0edc0ec..b5360b3 100644 --- a/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs +++ b/dotnet/src/Devolutions.Pinget.Publisher.Tests/PublisherPackagingTests.cs @@ -273,6 +273,84 @@ public void Publish_FailedPackagingTool_DoesNotExposeAnIndexOrRecordPublication( Assert.Equal(build.BuildId, new PublisherWorkspace(state).Status().LastPublishedBuildId); } + [Fact] + public void Publish_InitialPostCopyMsixVerificationFailure_CleansArtifactsAndAllowsRetry() + { + using var temp = new TempDirectory(); + var input = CopyFixture("multi", Path.Combine(temp.Path, "manifests")); + var state = Path.Combine(temp.Path, "state"); + var destination = Path.Combine(temp.Path, "published"); + var build = new PublisherWorkspace(state).Build(new PublisherBuildOptions(input, state)); + var makeAppx = PublisherPackaging.Doctor().MakeAppxPath; + Assert.False(string.IsNullOrWhiteSpace(makeAppx), "Windows SDK MakeAppx is required for the controlled post-copy failure case."); + + var wrapper = Path.Combine(temp.Path, "makeappx-fail-msix-identity.cmd"); + var manifestMutator = Path.Combine(temp.Path, "mutate-msix-identity.ps1"); + File.WriteAllText( + manifestMutator, + """ + $ErrorActionPreference = 'Stop' + $manifestPath = Join-Path $env:PINGET_PUBLISHER_TEST_PACKAGE_DIR 'AppxManifest.xml' + [xml]$manifest = Get-Content -LiteralPath $manifestPath -Raw + $identity = $manifest.SelectSingleNode('/*[local-name()=''Package'']/*[local-name()=''Identity'']') + if ($null -eq $identity) { + throw 'Generated package manifest does not contain an Identity element.' + } + $identity.SetAttribute('Name', 'Pinget.Publisher.WrongIdentity') + $manifest.Save($manifestPath) + """, + new UTF8Encoding(false)); + File.WriteAllText( + wrapper, + string.Join( + "\r\n", + [ + "@echo off", + "setlocal", + $"set \"REAL_MAKEAPPX={makeAppx}\"", + "set \"PINGET_PUBLISHER_TEST_PACKAGE_DIR=%~3\"", + "powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File \"%~dp0mutate-msix-identity.ps1\"", + "if errorlevel 1 exit /b 91", + "\"%REAL_MAKEAPPX%\" %*", + "exit /b %errorlevel%", + ]) + "\r\n", + new UTF8Encoding(false)); + + var failedPublish = new PublisherPublishOptions( + state, + destination, + PackageName, + Publisher, + "1.0.0.0", + UnsignedDevelopment: true, + MakeAppxPath: wrapper, + BuildId: build.BuildId); + var error = Assert.Throws(() => PublisherPackaging.Publish(failedPublish)); + + Assert.Contains("MSIX identity does not match", error.Message, StringComparison.Ordinal); + Assert.True(Directory.Exists(destination)); + Assert.Empty(Directory.EnumerateFileSystemEntries(destination)); + Assert.Null(new PublisherWorkspace(state).Status().LastPublishedBuildId); + + var snapshotFeed = Path.Combine(build.SnapshotDirectory, "feed"); + var copiedArtifactPaths = Directory.EnumerateFiles(snapshotFeed, "*", SearchOption.AllDirectories) + .Select(path => Path.GetRelativePath(snapshotFeed, path)) + .ToArray(); + Assert.NotEmpty(copiedArtifactPaths); + foreach (var relativePath in copiedArtifactPaths) + { + var destinationPath = Path.Combine(destination, relativePath.Replace('/', Path.DirectorySeparatorChar)); + Assert.False(File.Exists(destinationPath), $"Failed initial publication left copied artifact '{relativePath}'."); + } + + var retry = PublisherPackaging.Publish(UnsignedOptions(state, destination, "1.0.0.0", build.BuildId)); + + Assert.True(retry.Valid); + Assert.Equal(build.BuildId, retry.BuildId); + Assert.True(PublisherPackaging.VerifyDestination(destination).Valid); + Assert.Equal(build.BuildId, new PublisherWorkspace(state).Status().LastPublishedBuildId); + } + [Fact] public void Publish_FailedSignatureTool_DoesNotExposeAnIndexOrRecordPublication() { From 8efac12d9539554d0127bf679cf9f40658d99e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sat, 31 Oct 2026 14:21:12 -0400 Subject: [PATCH 3/8] Cover Rust initial publication rollback and restore release runner availability Inject a deterministic private post-copy failure to prove complete cleanup and successful retry, without changing verification ordering or public API. Gate the Windows-only lock test and migrate retiring macOS 14 release runners to macOS 15 with unchanged targets. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/release.yml | 4 +- docs/repository-publishing.md | 7 +- rust/crates/pinget-publisher/src/lib.rs | 114 +++++++++++++++++- .../pinget-publisher/tests/conformance.rs | 1 + 4 files changed, 120 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d028ec..71b3ee2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -190,12 +190,12 @@ jobs: archive_name: pinget-windows-arm64.zip binary_name: pinget.exe - name: macos-x64 - os: macos-14 + os: macos-15 target: x86_64-apple-darwin archive_name: pinget-macos-x64.zip binary_name: pinget - name: macos-arm64 - os: macos-14 + os: macos-15 target: aarch64-apple-darwin archive_name: pinget-macos-arm64.zip binary_name: pinget diff --git a/docs/repository-publishing.md b/docs/repository-publishing.md index e295a2e..ed4d7c0 100644 --- a/docs/repository-publishing.md +++ b/docs/repository-publishing.md @@ -196,8 +196,11 @@ Failure tests cover packaging/signing errors, update promotion rollback and destination locking. The .NET publisher also has an end-to-end regression for initial publication failure after metadata copying: a controlled MSIX identity mismatch leaves the destination empty and no publication recorded, -then a valid retry succeeds. The equivalent Rust post-copy failure path -remains unverified; Rust verifies the staged MSIX before copying metadata. +then a valid retry succeeds. The Rust regression uses a private deterministic +post-copy callback to verify the copied artifact hashes, inject a failure, +assert complete cleanup and unchanged publication state, then verify a normal +retry. Rust still verifies the staged MSIX before copying metadata; the +public API adds no failure-injection option. Stock-client acceptance and signature trust checks belong in a fresh Windows Sandbox or an ephemeral Windows CI runner. They must not register test sources diff --git a/rust/crates/pinget-publisher/src/lib.rs b/rust/crates/pinget-publisher/src/lib.rs index 96c849d..bd32548 100644 --- a/rust/crates/pinget-publisher/src/lib.rs +++ b/rust/crates/pinget-publisher/src/lib.rs @@ -660,6 +660,10 @@ pub fn verify_published_destination(destination: &Path, signtool_path: Option<&P /// Returns an error for unsupported hosts, invalid package identity/version, /// missing tools, failed signatures, unsafe destinations, or conflicting immutable files. pub fn publish(options: &PublishOptions) -> Result { + publish_after_copy(options, |_| Ok(())) +} + +fn publish_after_copy(options: &PublishOptions, after_copy: impl FnOnce(&Path) -> Result<()>) -> Result { if !cfg!(windows) { return Err(PublisherError::Tool( "MSIX packaging and signing are currently supported only on Windows".to_owned(), @@ -771,8 +775,8 @@ pub fn publish(options: &PublishOptions) -> Result { &destination, options, &build_id, - unsigned_development, prior, + after_copy, ); if result.is_err() { let _ = fs::remove_dir_all(&temporary); @@ -2110,9 +2114,10 @@ fn package_and_publish( destination: &Path, options: &PublishOptions, build_id: &str, - unsigned_development: bool, prior: Option, + after_copy: impl FnOnce(&Path) -> Result<()>, ) -> Result { + let unsigned_development = matches!(options.signing, Signing::UnsignedDevelopment); let metadata: SnapshotMetadata = serde_json::from_slice(&fs::read(snapshot.join("snapshot.json"))?)?; let package_root = temporary.join("package"); fs::create_dir_all(package_root.join("Public"))?; @@ -2219,6 +2224,7 @@ fn package_and_publish( } } } + after_copy(destination)?; let target_msix = destination.join("source2.msix"); let staged_package = stage_package(&msix, destination, build_id)?; rollback.files.push(staged_package.clone()); @@ -3830,6 +3836,110 @@ mod tests { fs::remove_dir_all(root).expect("cleanup"); } + #[cfg(windows)] + #[test] + fn initial_publish_post_copy_failure_cleans_destination_and_allows_retry() { + let makeappx = doctor().makeappx_path.expect("Windows SDK MakeAppx is required"); + let root = unit_directory("initial-publish"); + let input = root.join("input"); + let state = root.join("state"); + let destination = root.join("destination"); + fs::create_dir(&input).expect("manifest input"); + fs::write( + input.join("package.yaml"), + concat!( + "PackageIdentifier: Pinget.Test.InitialRollback\n", + "PackageVersion: 1.0.0\nPackageLocale: en-US\n", + "Publisher: Pinget Test\nPackageName: Initial Rollback\n", + "License: MIT\nShortDescription: Initial publication rollback regression\n", + "Installers:\n - Architecture: x64\n InstallerType: exe\n", + " InstallerUrl: https://example.invalid/unused.exe\n", + " InstallerSha256: 1111111111111111111111111111111111111111111111111111111111111111\n", + "ManifestType: singleton\nManifestVersion: 1.9.0\n", + ), + ) + .expect("manifest"); + let build_report = build(&BuildOptions { + input, + state: state.clone(), + }) + .expect("build snapshot"); + let metadata: SnapshotMetadata = serde_json::from_slice( + &fs::read(build_report.snapshot_directory.join("snapshot.json")).expect("snapshot metadata"), + ) + .expect("snapshot"); + assert!( + metadata.artifacts.len() >= 2, + "manifest and version sidecar must be copied" + ); + let options = PublishOptions { + state: state.clone(), + destination: destination.clone(), + build_id: Some(build_report.build_id.clone()), + name: "Pinget.Test.InitialRollback".to_owned(), + publisher: "CN=Pinget Initial Rollback Test".to_owned(), + version: "1.0.0.0".to_owned(), + signing: Signing::UnsignedDevelopment, + makeappx: Some(makeappx), + signtool: None, + timestamp_url: None, + }; + let mut reached_after_copy = false; + let error = publish_after_copy(&options, |feed| { + reached_after_copy = true; + for (relative, expected) in &metadata.artifacts { + let path = safe_join(feed, relative).expect("copied artifact path"); + assert!( + path.is_file(), + "artifact must exist before injected failure: {relative}" + ); + assert_eq!(sha256_hex(&fs::read(path).expect("copied artifact")), *expected); + } + assert!( + !feed.join(".pinget-publisher.json").exists(), + "no ownership marker before promotion" + ); + assert!(!feed.join("source2.msix").exists(), "no root package before promotion"); + Err(PublisherError::Invalid("injected initial post-copy failure".to_owned())) + }) + .expect_err("injected post-copy failure"); + assert!(reached_after_copy, "failure must occur after actual verified copies"); + assert!(error.to_string().contains("injected initial post-copy failure")); + assert_eq!( + fs::read_dir(&destination).expect("destination").count(), + 0, + "initial failure must leave no files, directories, marker, package or staging" + ); + let failed_status = status(&state).expect("state after failure"); + assert_eq!(failed_status.last_published_build_id, None); + assert!(!failed_status.last_build_published); + verify(&state, Some(&build_report.build_id), None).expect("snapshot preserved after failure"); + + let published = publish(&options).expect("normal initial retry"); + assert!(published.valid); + assert!( + !published.reused, + "retry must perform publication rather than reuse a false commit" + ); + assert_eq!(published.build_id, build_report.build_id); + let verified = verify_published_destination(&destination, None).expect("destination verification"); + assert!(verified.valid); + assert_eq!(verified.build_id.as_deref(), Some(build_report.build_id.as_str())); + assert_eq!( + verified.artifact_count, + metadata.artifacts.len() + 1, + "feed artifacts plus root package" + ); + let published_status = status(&state).expect("published state"); + assert_eq!( + published_status.last_published_build_id.as_deref(), + Some(build_report.build_id.as_str()) + ); + assert!(published_status.last_build_published); + verify(&state, Some(&build_report.build_id), None).expect("snapshot preserved after retry"); + fs::remove_dir_all(root).expect("cleanup"); + } + #[test] fn destination_lock_serializes_independent_publisher_states() { let root = unit_directory("lock"); diff --git a/rust/crates/pinget-publisher/tests/conformance.rs b/rust/crates/pinget-publisher/tests/conformance.rs index 95351d9..e3dcea0 100644 --- a/rust/crates/pinget-publisher/tests/conformance.rs +++ b/rust/crates/pinget-publisher/tests/conformance.rs @@ -1132,6 +1132,7 @@ fn publish_semantic_metadata_promotion_failure_preserves_old_feed_then_retry_suc } #[test] +#[cfg(windows)] fn publish_destination_lock_serializes_different_state_roots_and_allows_retry() { use std::os::windows::fs::OpenOptionsExt; From 019055c9d4bed6656bc22cec836f50e20cdcd76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sat, 31 Oct 2026 14:35:47 -0400 Subject: [PATCH 4/8] Run signed publisher acceptance on disposable Windows CI runners Use runner-local short-lived signing and TLS certificates, isolated source names and verified cleanup without Sandbox or repository secrets. Provision PyYAML for independent conformance and migrate CLI packaging off retired macOS 14 runners. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 78 +++- .github/workflows/cli-packages.yml | 4 +- .../run-ephemeral-host-publisher-gate.ps1 | 343 ++++++++++++++++++ 3 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 tests/publisher/run-ephemeral-host-publisher-gate.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47a0e4d..bbfd4e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,15 @@ jobs: - name: Run publisher tests run: dotnet test dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj -c Release - - name: Check Rust and .NET publisher interoperability + - name: Setup Python for independent publisher validation + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install independent publisher validator dependency + run: python -m pip install "PyYAML>=6,<7" + + - name: Run Rust and .NET publisher interoperability shell: pwsh run: | cargo build --manifest-path rust/Cargo.toml -p pinget-cli --release @@ -157,3 +165,71 @@ jobs: - name: Validate PowerShell module packaging shell: pwsh run: pwsh -NoLogo -NoProfile -File (Resolve-Path 'scripts/Build-PowerShellModule.ps1') -NoBuild -OutputRoot artifacts/ci-powershell -Clean + + stock-winget-acceptance: + name: Signed stock WinGet acceptance (ephemeral runner) + runs-on: windows-latest + needs: + - dotnet + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + - name: Setup Rust + shell: pwsh + run: | + rustup toolchain install stable --profile minimal --target x86_64-pc-windows-msvc + rustup default stable + + - name: Build disposable x64 publisher CLIs + shell: pwsh + run: | + pwsh -NoLogo -NoProfile -File (Resolve-Path 'scripts/Build-CliNativeNuGetPackages.ps1') ` + -Package All ` + -RustRuntimeIdentifiers 'win-x64' ` + -DotNetRuntimeIdentifiers 'win-x64' ` + -NoPack ` + -Clean ` + -StagingRoot artifacts/ci-stock-winget ` + -OutputRoot artifacts/ci-stock-winget-packages + if ($LASTEXITCODE -ne 0) { throw 'Building the disposable x64 publisher CLIs failed.' } + + - name: Run signed stock WinGet acceptance with runner-local credentials + shell: pwsh + run: | + $sdkRoot = Join-Path ${env:ProgramFiles(x86)} 'Windows Kits\10\bin' + $sdkTools = Get-ChildItem -LiteralPath $sdkRoot -Directory | + Sort-Object Name -Descending | + ForEach-Object { + [pscustomobject]@{ + MakeAppx = Join-Path $_.FullName 'x64\makeappx.exe' + SignTool = Join-Path $_.FullName 'x64\signtool.exe' + } + } | + Where-Object { + (Test-Path -LiteralPath $_.MakeAppx) -and + (Test-Path -LiteralPath $_.SignTool) + } | + Select-Object -First 1 + if (-not $sdkTools) { throw 'Windows SDK MakeAppx and SignTool were not found.' } + + pwsh -NoLogo -NoProfile -File (Resolve-Path 'tests/publisher/run-ephemeral-host-publisher-gate.ps1') ` + -RustCliPath (Resolve-Path 'artifacts/ci-stock-winget/rust/win-x64/pinget.exe') ` + -DotnetCliPath (Resolve-Path 'artifacts/ci-stock-winget/dotnet-nativeaot/win-x64/pinget.exe') ` + -MakeAppxPath $sdkTools.MakeAppx ` + -SignToolPath $sdkTools.SignTool + if ($LASTEXITCODE -ne 0) { throw 'Signed stock WinGet acceptance failed.' } + + - name: Remove disposable build outputs + if: always() + shell: pwsh + run: | + Remove-Item -LiteralPath 'artifacts/ci-stock-winget' -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath 'artifacts/ci-stock-winget-packages' -Recurse -Force -ErrorAction SilentlyContinue diff --git a/.github/workflows/cli-packages.yml b/.github/workflows/cli-packages.yml index 2f50202..1196f1c 100644 --- a/.github/workflows/cli-packages.yml +++ b/.github/workflows/cli-packages.yml @@ -41,10 +41,10 @@ jobs: os: windows-latest rust-target: aarch64-pc-windows-msvc - rid: osx-x64 - os: macos-14 + os: macos-15 rust-target: x86_64-apple-darwin - rid: osx-arm64 - os: macos-14 + os: macos-15 rust-target: aarch64-apple-darwin steps: diff --git a/tests/publisher/run-ephemeral-host-publisher-gate.ps1 b/tests/publisher/run-ephemeral-host-publisher-gate.ps1 new file mode 100644 index 0000000..7506a1d --- /dev/null +++ b/tests/publisher/run-ephemeral-host-publisher-gate.ps1 @@ -0,0 +1,343 @@ +param( + [Parameter(Mandatory = $true)] + [string] $RustCliPath, + [Parameter(Mandatory = $true)] + [string] $DotnetCliPath, + [Parameter(Mandatory = $true)] + [string] $MakeAppxPath, + [Parameter(Mandatory = $true)] + [string] $SignToolPath +) + +$ErrorActionPreference = 'Stop' + +if ($env:GITHUB_ACTIONS -ne 'true' -or + $env:RUNNER_ENVIRONMENT -ne 'github-hosted' -or + [string]::IsNullOrWhiteSpace($env:RUNNER_TEMP) -or + -not [System.IO.Path]::IsPathRooted($env:RUNNER_TEMP)) { + throw 'This gate is restricted to a GitHub-hosted runner with an absolute RUNNER_TEMP directory.' +} +if (-not [System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform( + [System.Runtime.InteropServices.OSPlatform]::Windows)) { + throw 'The signed stock WinGet gate requires a Windows GitHub-hosted runner.' +} + +$identity = [System.Security.Principal.WindowsIdentity]::GetCurrent() +$principal = [System.Security.Principal.WindowsPrincipal]::new($identity) +if (-not $principal.IsInRole([System.Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'The GitHub-hosted Windows runner must permit runner-local LocalMachine Root certificate trust.' +} + +foreach ($path in @($RustCliPath, $DotnetCliPath, $MakeAppxPath, $SignToolPath)) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required publisher or Windows SDK tool was not found: $path" + } +} + +$appInstaller = Get-AppxPackage -Name Microsoft.DesktopAppInstaller +if ($null -eq $appInstaller) { + throw 'The GitHub-hosted Windows runner does not have the stock Microsoft.DesktopAppInstaller package.' +} +$wingetCommand = Get-Command winget.exe -ErrorAction SilentlyContinue +if ($null -eq $wingetCommand) { + throw 'The GitHub-hosted Windows runner does not expose the stock winget.exe execution alias.' +} + +$workRoot = Join-Path $env:RUNNER_TEMP "pinget-stock-gate-$([Guid]::NewGuid().ToString('N'))" +$signerPfxPath = Join-Path $workRoot 'publisher-signing.pfx' +$signerCertificatePath = Join-Path $workRoot 'publisher-signing.cer' +$tlsPfxPath = Join-Path $workRoot 'tls-server.pfx' +$tlsCertificatePath = Join-Path $workRoot 'tls-server.cer' +$passwordVariable = 'PINGET_PUBLISHER_EPHEMERAL_PASSWORD' +$password = $null +$signerThumbprint = $null +$tlsThumbprint = $null +$trustMayHaveBeenAdded = $false +$gateStarted = $false +$gateSucceeded = $false +$workDirectoryCreated = $false +$sourceName = $null +$wingetPath = $wingetCommand.Source +$publisherSubject = 'CN=Pinget Publisher Conformance Test' + +function New-RandomPassword { + $bytes = [byte[]]::new(32) + [System.Security.Cryptography.RandomNumberGenerator]::Fill($bytes) + try { + return [Convert]::ToBase64String($bytes) + } + finally { + [System.Security.Cryptography.CryptographicOperations]::ZeroMemory($bytes) + } +} + +function Get-RootStoreThumbprints { + $store = [System.Security.Cryptography.X509Certificates.X509Store]::new( + [System.Security.Cryptography.X509Certificates.StoreName]::Root, + [System.Security.Cryptography.X509Certificates.StoreLocation]::LocalMachine) + try { + $store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadOnly) + return @($store.Certificates | ForEach-Object Thumbprint) + } + finally { + $store.Close() + } +} + +function Test-RunWinGetSourceRegistered { + $escapedSourceName = [regex]::Escape($sourceName) + $sourceList = & $wingetPath source list --disable-interactivity 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Could not inspect WinGet sources: $($sourceList -join "`n")" + } + + # `winget source list` renders a table with URL/type columns after the + # source name; matching a name-only line misses registered sources. + return ($sourceList -join "`n") -match "(?m)^\s*$escapedSourceName\s+\S" +} + +function Remove-RunRootCertificates { + $thumbprints = @($signerThumbprint, $tlsThumbprint) | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + if ($thumbprints.Count -eq 0) { + return + } + + $store = [System.Security.Cryptography.X509Certificates.X509Store]::new( + [System.Security.Cryptography.X509Certificates.StoreName]::Root, + [System.Security.Cryptography.X509Certificates.StoreLocation]::LocalMachine) + try { + $store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite) + foreach ($certificate in @($store.Certificates)) { + if ($thumbprints -contains $certificate.Thumbprint) { + $store.Remove($certificate) + $certificate.Dispose() + } + } + } + finally { + $store.Close() + } +} + +function Remove-RunWinGetSource { + if (-not $gateStarted -or -not (Test-Path -LiteralPath $wingetPath)) { + return + } + + if (Test-RunWinGetSourceRegistered) { + $removeOutput = & $wingetPath source remove --name $sourceName --disable-interactivity 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Could not remove temporary WinGet source '$sourceName': $($removeOutput -join "`n")" + } + } + + if (Test-RunWinGetSourceRegistered) { + throw "Temporary WinGet source '$sourceName' remains registered after cleanup." + } +} + +try { + New-Item -ItemType Directory -Path $workRoot -ErrorAction Stop | Out-Null + $workDirectoryCreated = $true + + $fixtureDirectory = Join-Path $PSScriptRoot 'fixtures\singleton' + if (-not (Test-Path -LiteralPath $fixtureDirectory -PathType Container)) { + throw "Committed singleton stock-client fixture directory is missing: $fixtureDirectory" + } + + $password = New-RandomPassword + [Environment]::SetEnvironmentVariable($passwordVariable, $password, 'Process') + + $signerRsa = [System.Security.Cryptography.RSA]::Create(2048) + try { + $signerRequest = [System.Security.Cryptography.X509Certificates.CertificateRequest]::new( + $publisherSubject, + $signerRsa, + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1) + $signerRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509KeyUsageExtension]::new( + [System.Security.Cryptography.X509Certificates.X509KeyUsageFlags]::DigitalSignature, + $true)) + $signerEku = [System.Security.Cryptography.OidCollection]::new() + [void]$signerEku.Add([System.Security.Cryptography.Oid]::new('1.3.6.1.5.5.7.3.3')) + $signerRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509EnhancedKeyUsageExtension]::new( + $signerEku, $false)) + $signerCertificate = $signerRequest.CreateSelfSigned( + [DateTimeOffset]::UtcNow.AddMinutes(-5), + [DateTimeOffset]::UtcNow.AddDays(1)) + try { + $signerThumbprint = $signerCertificate.Thumbprint + [System.IO.File]::WriteAllBytes( + $signerPfxPath, + $signerCertificate.Export( + [System.Security.Cryptography.X509Certificates.X509ContentType]::Pfx, + $password)) + [System.IO.File]::WriteAllBytes( + $signerCertificatePath, + $signerCertificate.Export( + [System.Security.Cryptography.X509Certificates.X509ContentType]::Cert)) + } + finally { + $signerCertificate.Dispose() + } + } + finally { + $signerRsa.Dispose() + } + + $tlsRsa = [System.Security.Cryptography.RSA]::Create(2048) + try { + $tlsRequest = [System.Security.Cryptography.X509Certificates.CertificateRequest]::new( + 'CN=127.0.0.1', + $tlsRsa, + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1) + $tlsRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509BasicConstraintsExtension]::new( + $false, $false, 0, $true)) + $tlsRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509KeyUsageExtension]::new( + [System.Security.Cryptography.X509Certificates.X509KeyUsageFlags]::DigitalSignature, + $true)) + $tlsEku = [System.Security.Cryptography.OidCollection]::new() + [void]$tlsEku.Add([System.Security.Cryptography.Oid]::new('1.3.6.1.5.5.7.3.1')) + $tlsRequest.CertificateExtensions.Add( + [System.Security.Cryptography.X509Certificates.X509EnhancedKeyUsageExtension]::new( + $tlsEku, $false)) + $san = [System.Security.Cryptography.X509Certificates.SubjectAlternativeNameBuilder]::new() + $san.AddIpAddress([System.Net.IPAddress]::Parse('127.0.0.1')) + $tlsRequest.CertificateExtensions.Add($san.Build()) + $tlsCertificate = $tlsRequest.CreateSelfSigned( + [DateTimeOffset]::UtcNow.AddMinutes(-5), + [DateTimeOffset]::UtcNow.AddDays(1)) + try { + $tlsThumbprint = $tlsCertificate.Thumbprint + [System.IO.File]::WriteAllBytes( + $tlsPfxPath, + $tlsCertificate.Export( + [System.Security.Cryptography.X509Certificates.X509ContentType]::Pfx)) + [System.IO.File]::WriteAllBytes( + $tlsCertificatePath, + $tlsCertificate.Export( + [System.Security.Cryptography.X509Certificates.X509ContentType]::Cert)) + } + finally { + $tlsCertificate.Dispose() + } + } + finally { + $tlsRsa.Dispose() + } + + $existingThumbprints = Get-RootStoreThumbprints + if ($existingThumbprints -contains $signerThumbprint -or + $existingThumbprints -contains $tlsThumbprint) { + throw 'A generated disposable certificate unexpectedly already exists in LocalMachine Root; refusing to alter pre-existing trust.' + } + $harness = Join-Path $PSScriptRoot 'sandbox-conformance.ps1' + $publishers = @( + [pscustomobject]@{ Name = 'Rust'; Cli = $RustCliPath }, + [pscustomobject]@{ Name = 'Dotnet'; Cli = $DotnetCliPath } + ) + foreach ($publisher in $publishers) { + $publisherWorkRoot = Join-Path $workRoot $publisher.Name.ToLowerInvariant() + $inputDirectory = Join-Path $publisherWorkRoot 'input' + $stateDirectory = Join-Path $publisherWorkRoot 'publisher-state' + $sourceName = "pinget-ci-$($publisher.Name.ToLowerInvariant())-$([Guid]::NewGuid().ToString('N'))" + $escapedSourceName = [regex]::Escape($sourceName) + New-Item -ItemType Directory -Path $publisherWorkRoot -ErrorAction Stop | Out-Null + Copy-Item -LiteralPath $fixtureDirectory -Destination $inputDirectory -Recurse + Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'serve-feed.ps1') ` + -Destination (Join-Path $publisherWorkRoot 'serve-feed.ps1') + Copy-Item -LiteralPath $tlsPfxPath -Destination (Join-Path $publisherWorkRoot 'tls-server.pfx') + Copy-Item -LiteralPath $tlsCertificatePath -Destination (Join-Path $publisherWorkRoot 'tls-server.cer') + + $existingSources = & $wingetPath source list --disable-interactivity 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Could not inspect existing WinGet sources: $($existingSources -join "`n")" + } + if (($existingSources -join "`n") -match "(?m)^\s*$escapedSourceName\s+\S") { + throw "Generated temporary source name unexpectedly collides with existing source '$sourceName'." + } + + $trustMayHaveBeenAdded = $true + $gateStarted = $true + Write-Output "START signed stock WinGet acceptance for $($publisher.Name)." + & $harness ` + -WorkRoot $publisherWorkRoot ` + -SourceName $sourceName ` + -ExecutionTarget EphemeralHost ` + -AllowEphemeralHostMutations ` + -ExpectedWinGetVersion $appInstaller.Version.ToString() ` + -SignerCertificatePath $signerCertificatePath ` + -PublisherCliPath $publisher.Cli ` + -PublisherStateDirectory $stateDirectory ` + -PublisherInputDirectory $inputDirectory ` + -PfxPath $signerPfxPath ` + -PfxPasswordEnvironmentVariable $passwordVariable ` + -RustReaderCliPath $RustCliPath ` + -DotnetReaderCliPath $DotnetCliPath ` + -PublisherName 'Pinget.Publisher.Conformance' ` + -PublisherIdentity $publisherSubject ` + -InitialPackageVersion '1.0.0.0' ` + -MakeAppxPath $MakeAppxPath ` + -SignToolPath $SignToolPath + + Remove-RunWinGetSource + $gateStarted = $false + $remainingThumbprints = Get-RootStoreThumbprints + if ($remainingThumbprints -contains $signerThumbprint -or + $remainingThumbprints -contains $tlsThumbprint) { + throw "The $($publisher.Name) gate left a temporary certificate in LocalMachine Root." + } + $trustMayHaveBeenAdded = $false + } + + $gateSucceeded = $true + Write-Output "PASS signed stock WinGet conformance for Rust and .NET on GitHub runner with WinGet $($appInstaller.Version)." +} +finally { + $cleanupErrors = [System.Collections.Generic.List[string]]::new() + if ($gateStarted) { + try { + Remove-RunWinGetSource + } + catch { + $cleanupErrors.Add($_.Exception.Message) + } + } + if ($trustMayHaveBeenAdded) { + try { + Remove-RunRootCertificates + $remainingThumbprints = Get-RootStoreThumbprints + if ($remainingThumbprints -contains $signerThumbprint -or + $remainingThumbprints -contains $tlsThumbprint) { + throw 'Disposable certificate cleanup verification found a test certificate still trusted.' + } + } + catch { + $cleanupErrors.Add($_.Exception.Message) + } + } + + [Environment]::SetEnvironmentVariable($passwordVariable, $null, 'Process') + $password = $null + if ($workDirectoryCreated -and (Test-Path -LiteralPath $workRoot)) { + try { + Remove-Item -LiteralPath $workRoot -Recurse -Force + } + catch { + $cleanupErrors.Add("Could not remove runner-local gate directory '$workRoot': $($_.Exception.Message)") + } + } + + if ($cleanupErrors.Count -gt 0) { + throw "Ephemeral stock WinGet gate cleanup failed: $($cleanupErrors -join '; ')" + } + if (-not $gateSucceeded) { + Write-Output 'Ephemeral stock WinGet gate did not pass; runner-local cleanup completed.' + } +} From 34c1f07386fb3164a0cfe0db5ee1c3e244720673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sat, 31 Oct 2026 14:36:10 -0400 Subject: [PATCH 5/8] Separate native publisher fixtures from cross-platform conformance tests Keep inventory diff, manifest validation and state/path checks active on non-Windows hosts, add an explicit native-compression capability regression, and gate tests needing Windows MSZIP generation. Verify both Windows and Ubuntu WSL suites. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../pinget-publisher/tests/conformance.rs | 229 ++++++++++++++---- 1 file changed, 185 insertions(+), 44 deletions(-) diff --git a/rust/crates/pinget-publisher/tests/conformance.rs b/rust/crates/pinget-publisher/tests/conformance.rs index e3dcea0..8235884 100644 --- a/rust/crates/pinget-publisher/tests/conformance.rs +++ b/rust/crates/pinget-publisher/tests/conformance.rs @@ -1,16 +1,21 @@ +use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use pinget_publisher::{ - BuildOptions, PublishOptions, Signing, build, diff, doctor, publish, status, verify, verify_published_destination, -}; +#[cfg(not(windows))] +use pinget_publisher::PublisherError; +use pinget_publisher::{BuildOptions, build, diff, status, verify}; +#[cfg(windows)] +use pinget_publisher::{PublishOptions, Signing, doctor, publish, verify_published_destination}; use rusqlite::Connection; +#[cfg(windows)] use serde_json::Value; static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); #[test] +#[cfg(windows)] fn build_multifile_manifest_set_emits_complete_v2_index_and_feed() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -200,6 +205,7 @@ fn build_multifile_manifest_set_emits_complete_v2_index_and_feed() { } #[test] +#[cfg(windows)] fn build_uses_declared_default_locale_and_embeds_other_localizations_once() { let temp = TempDirectory::new(); let input = temp.path.join("manifests"); @@ -250,6 +256,7 @@ fn build_uses_declared_default_locale_and_embeds_other_localizations_once() { } #[test] +#[cfg(windows)] fn build_preserves_schema_string_scalars_in_canonical_manifest_output() { let temp = TempDirectory::new(); let input = temp.path.join("manifests"); @@ -311,6 +318,7 @@ ManifestVersion: 1.9.0 } #[test] +#[cfg(windows)] fn equivalent_input_trees_produce_the_same_build_identifier() { let temp = TempDirectory::new(); let first_input = copy_fixture("multi", &temp.path.join("first")); @@ -334,11 +342,8 @@ fn diff_reports_raw_content_additions_changes_and_removals() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); let state = temp.path.join("state"); - build(&BuildOptions { - input: input.clone(), - state: state.clone(), - }) - .expect("initial fixture should build"); + let build_id = create_inventory_state(&input, &state); + let original_database = fs::read(state.join("publisher.db")).expect("inventory state"); let unchanged = diff(&BuildOptions { input: input.clone(), state: state.clone(), @@ -347,6 +352,7 @@ fn diff_reports_raw_content_additions_changes_and_removals() { assert!(unchanged.added.is_empty()); assert!(unchanged.changed.is_empty()); assert!(unchanged.removed.is_empty()); + assert_eq!(unchanged.last_build_id.as_deref(), Some(build_id.as_str())); let changed_manifest = input.join("Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"); let mut changed_content = fs::read_to_string(&changed_manifest).expect("fixture should be readable"); @@ -359,11 +365,81 @@ fn diff_reports_raw_content_additions_changes_and_removals() { .expect("new manifest should be written"); fs::remove_file(input.join("Pinget.Test.Matrix_1.9.0.yaml")).expect("old version fixture should be removed"); - let changes = diff(&BuildOptions { input, state }).expect("changed input should diff"); + let changes = diff(&BuildOptions { + input, + state: state.clone(), + }) + .expect("changed input should diff"); assert_eq!(changes.added, ["Pinget.Test.Added.yaml"]); assert_eq!(changes.changed, ["Pinget.Test.Matrix_1.10.0.locale.en-US.yaml"]); assert_eq!(changes.removed, ["Pinget.Test.Matrix_1.9.0.yaml"]); assert!(!changes.last_build_published); + assert_eq!( + fs::read(state.join("publisher.db")).expect("unchanged inventory state"), + original_database + ); + assert!( + !state.join("builds").exists(), + "diff must not require or generate native artifacts" + ); +} + +#[cfg(not(windows))] +#[test] +fn non_windows_build_rejects_native_compression_while_diff_succeeds() { + let temp = TempDirectory::new(); + let input = copy_fixture("multi", &temp.path.join("manifests")); + let failed_state = temp.path.join("native-state"); + let error = build(&BuildOptions { + input: input.clone(), + state: failed_state.clone(), + }) + .expect_err("native writer must not claim support on a non-Windows host"); + match error { + PublisherError::Tool(message) => { + assert!(message.contains("native WinGet MSZIP compression")); + assert!(message.contains("only on Windows")); + } + other => panic!("expected native capability error, got {other}"), + } + assert!( + !failed_state.join("publisher.db").exists(), + "failed native build must not record a snapshot" + ); + assert_eq!( + fs::read_dir(failed_state.join("builds")) + .expect("snapshot directory") + .count(), + 0, + "failed native build must clean private staging" + ); + let state = temp.path.join("inventory-state"); + create_empty_state(&state); + let original = fs::read(state.join("publisher.db")).expect("inventory state"); + let changes = diff(&BuildOptions { + input, + state: state.clone(), + }) + .expect("diff must not call native compression"); + assert_eq!( + changes.added, + [ + "Pinget.Test.Matrix_1.10.0.installer.yaml", + "Pinget.Test.Matrix_1.10.0.locale.en-US.yaml", + "Pinget.Test.Matrix_1.10.0.locale.fr-FR.yaml", + "Pinget.Test.Matrix_1.10.0.version.yaml", + "Pinget.Test.Matrix_1.9.0.yaml", + ] + ); + assert!(changes.changed.is_empty()); + assert!(changes.removed.is_empty()); + assert_eq!(changes.last_build_id, None); + assert!(!changes.last_build_published); + assert_eq!( + fs::read(state.join("publisher.db")).expect("inventory state preserved"), + original + ); + assert!(!state.join("builds").exists()); } #[test] @@ -386,18 +462,17 @@ fn readonly_state_commands_do_not_initialize_missing_state() { #[test] fn read_only_operations_reject_reparse_points_and_state_overlap() { - #[cfg(windows)] + #[cfg(any(windows, unix))] { + #[cfg(unix)] + use std::os::unix::fs::symlink as symlink_dir; + #[cfg(windows)] use std::os::windows::fs::symlink_dir; let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); let state = temp.path.join("state"); - let report = build(&BuildOptions { - input: input.clone(), - state: state.clone(), - }) - .expect("fixture should build before reparse checks"); + create_empty_state(&state); let input_link = temp.path.join("manifests-link"); let state_link = temp.path.join("state-link"); symlink_dir(&input, &input_link).expect("input directory symlink should be available"); @@ -406,29 +481,45 @@ fn read_only_operations_reject_reparse_points_and_state_overlap() { input: input_link, state, }; - assert!(diff(&diff_options).is_err()); - assert!(status(&state_link).is_err()); - assert!(verify(&state_link, Some(&report.build_id), None).is_err()); - fs::remove_dir(&diff_options.input).expect("input symlink should be removed"); - fs::remove_dir(state_link).expect("state symlink should be removed"); + for error in [ + diff(&diff_options).expect_err("diff must reject symlink input"), + status(&state_link).expect_err("status must reject symlink state"), + verify(&state_link, None, None).expect_err("verify must reject symlink state"), + ] { + assert!( + error.to_string().contains("reparse points"), + "unexpected symlink rejection: {error}" + ); + } + #[cfg(windows)] + { + fs::remove_dir(&diff_options.input).expect("input directory symlink should be removed"); + fs::remove_dir(state_link).expect("state directory symlink should be removed"); + } + #[cfg(unix)] + { + fs::remove_file(&diff_options.input).expect("input symlink should be unlinked"); + fs::remove_file(state_link).expect("state symlink should be unlinked"); + } } let temp = TempDirectory::new(); - let input = copy_fixture("multi", &temp.path.join("manifests")); let state = temp.path.join("state"); - let report = build(&BuildOptions { - input, - state: state.clone(), - }) - .expect("fixture should build before overlap check"); - let nested_input = state.join("builds").join(&report.build_id).join("feed"); + create_empty_state(&state); + let original_database = fs::read(state.join("publisher.db")).expect("state database"); + let nested_input = state.join("nested-input"); + fs::create_dir(&nested_input).expect("overlapping input directory"); let error = diff(&BuildOptions { input: nested_input, state: state.clone(), }) .expect_err("diff must reject an input root nested inside its state root"); assert!(error.to_string().to_ascii_lowercase().contains("overlap")); - assert!(verify(&state, Some(&report.build_id), None).is_ok()); + assert_eq!( + fs::read(state.join("publisher.db")).expect("preserved state"), + original_database + ); + assert_eq!(status(&state).expect("preserved valid state").build_count, 0); } #[test] @@ -436,11 +527,7 @@ fn build_rejects_future_sqlite_user_version_without_mutating_state() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); let state = temp.path.join("state"); - build(&BuildOptions { - input: input.clone(), - state: state.clone(), - }) - .expect("initial fixture should create state"); + create_empty_state(&state); let database_path = state.join("publisher.db"); { let database = Connection::open(&database_path).expect("state database should open"); @@ -466,11 +553,7 @@ fn build_rejects_legacy_artifact_format_state_without_mutating_it() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); let state = temp.path.join("state"); - build(&BuildOptions { - input: input.clone(), - state: state.clone(), - }) - .expect("current-format fixture should create state"); + create_empty_state(&state); let database_path = state.join("publisher.db"); { let database = Connection::open(&database_path).expect("state database should open"); @@ -509,6 +592,7 @@ fn build_rejects_legacy_artifact_format_state_without_mutating_it() { } #[test] +#[cfg(windows)] fn verify_rejects_legacy_artifact_format_snapshot_without_mutating_it() { for format_version in [None, Some(1), Some(999)] { let temp = TempDirectory::new(); @@ -578,6 +662,7 @@ fn build_rejects_foreign_sqlite_state_without_mutating_it() { } #[test] +#[cfg(windows)] fn build_preserves_stale_staging_owned_by_another_attempt_and_completes_snapshot() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -731,6 +816,7 @@ fn existing_writer_lock_prevents_build_mutation() { } #[test] +#[cfg(windows)] fn publish_unsigned_is_idempotent_and_excludes_staging_files_from_inventory() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -788,6 +874,7 @@ fn publish_unsigned_is_idempotent_and_excludes_staging_files_from_inventory() { } #[test] +#[cfg(windows)] fn publish_rejects_downgrade_and_retains_immutable_artifacts_on_update() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -872,6 +959,7 @@ fn publish_rejects_downgrade_and_retains_immutable_artifacts_on_update() { } #[test] +#[cfg(windows)] fn publish_rejects_unowned_destination_without_overwriting_unrelated_file() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -911,6 +999,7 @@ fn publish_rejects_unowned_destination_without_overwriting_unrelated_file() { } #[test] +#[cfg(windows)] fn publish_rejects_identity_change_and_untracked_artifact_without_mutation() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -988,6 +1077,7 @@ fn publish_rejects_identity_change_and_untracked_artifact_without_mutation() { } #[test] +#[cfg(windows)] fn publish_failed_packaging_tool_does_not_publish_or_record_build() { let temp = TempDirectory::new(); let input = copy_fixture("multi", &temp.path.join("manifests")); @@ -1221,6 +1311,7 @@ fn publish_destination_lock_serializes_different_state_roots_and_allows_retry() } #[test] +#[cfg(windows)] fn verify_rejects_corrupted_snapshot_sidecar_and_index() { for (component, location) in [ ("snapshot", ""), @@ -1259,13 +1350,8 @@ fn verify_rejects_corrupted_snapshot_sidecar_and_index() { #[test] fn status_rejects_corrupted_state_database_without_reinitializing_it() { let temp = TempDirectory::new(); - let input = copy_fixture("multi", &temp.path.join("manifests")); let state = temp.path.join("state"); - build(&BuildOptions { - input, - state: state.clone(), - }) - .expect("fixture should build before state corruption"); + create_empty_state(&state); let database = state.join("publisher.db"); let corrupted = b"not a sqlite database"; fs::write(&database, corrupted).expect("state database should be corrupted"); @@ -1277,6 +1363,54 @@ fn status_rejects_corrupted_state_database_without_reinitializing_it() { ); } +fn create_empty_state(state: &Path) { + fs::create_dir_all(state).expect("inventory state directory"); + let database = Connection::open(state.join("publisher.db")).expect("inventory state database"); + // Read-only inventory operations require state records, not native index/feed artifacts. + database + .execute_batch( + "PRAGMA user_version=1; + CREATE TABLE metadata(name TEXT PRIMARY KEY NOT NULL,value TEXT NOT NULL) WITHOUT ROWID; + CREATE TABLE builds(id TEXT PRIMARY KEY NOT NULL,inventory TEXT NOT NULL,summary TEXT NOT NULL) WITHOUT ROWID; + INSERT INTO metadata VALUES('schemaVersion','1'),('artifactFormatVersion','2');", + ) + .expect("independent state fixture schema"); +} + +fn create_inventory_state(input: &Path, state: &Path) -> String { + create_empty_state(state); + let initial = diff(&BuildOptions { + input: input.to_path_buf(), + state: state.to_path_buf(), + }) + .expect("scan manifest inventory without compression"); + let inventory: BTreeMap<_, _> = initial + .added + .into_iter() + .map(|relative| { + let hash = sha256_file(&input.join(&relative)); + (relative, hash) + }) + .collect(); + let inventory = serde_json::to_string(&inventory).expect("inventory JSON"); + let build_id = sha256_hex(format!("pinget-publisher-state-v2\n{inventory}").as_bytes()); + let database = Connection::open(state.join("publisher.db")).expect("inventory database"); + database + .execute( + "INSERT INTO builds(id,inventory,summary) VALUES(?1,?2,?3)", + rusqlite::params![ + build_id, + inventory, + r#"{"createdUtc":"2026-01-01T00:00:00Z","manifestCount":5,"packageCount":1}"# + ], + ) + .expect("independent inventory record"); + database + .execute("INSERT INTO metadata VALUES('lastBuildId',?1)", [&build_id]) + .expect("latest inventory record"); + build_id +} + fn apply_mutation(input: &Path, mutation: &str) { let version = input.join("Pinget.Test.Matrix_1.10.0.version.yaml"); match mutation { @@ -1346,6 +1480,7 @@ fn apply_mutation(input: &Path, mutation: &str) { } } +#[cfg(windows)] fn unsigned_publish_options(state: &Path, destination: &Path, version: &str, build_id: &str) -> PublishOptions { let makeappx = makeappx_tool(); PublishOptions { @@ -1362,6 +1497,7 @@ fn unsigned_publish_options(state: &Path, destination: &Path, version: &str, bui } } +#[cfg(windows)] fn makeappx_tool() -> PathBuf { if let Some(path) = doctor().makeappx_path.filter(|path| path.is_file()) { return path; @@ -1404,6 +1540,7 @@ fn singleton_manifest(id: &str, version: &str) -> String { ) } +#[cfg(windows)] fn write_locale_order_manifests(input: &Path, id: &str) { for (suffix, locale, publisher, name, kind) in [ ( @@ -1487,6 +1624,7 @@ fn copy_directory(source: &Path, destination: &Path) { } } +#[cfg(windows)] fn find_first_file(directory: &Path) -> PathBuf { let mut pending = vec![directory.to_path_buf()]; while let Some(current) = pending.pop() { @@ -1502,6 +1640,7 @@ fn find_first_file(directory: &Path) -> PathBuf { panic!("expected at least one snapshot file"); } +#[cfg(windows)] fn count_files_with_extension(directory: &Path, extension: &str) -> usize { let mut pending = vec![directory.to_path_buf()]; let mut count = 0; @@ -1528,6 +1667,7 @@ fn sha256_hex(bytes: &[u8]) -> String { encode_hex(&Sha256::digest(bytes)) } +#[cfg(windows)] fn read_correlations(database: &Connection, table: &str, column: &str) -> Vec { let query = format!( "SELECT {column} FROM {table} AS valuesTable \ @@ -1542,6 +1682,7 @@ fn read_correlations(database: &Connection, table: &str, column: &str) -> Vec Date: Sat, 31 Oct 2026 21:44:32 -0400 Subject: [PATCH 6/8] Validate WinGet client and App Installer versions independently Hosted runners can ship an App Installer package whose version differs from its winget executable. Capture and validate the actual CLI version separately while retaining the package version check. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/publisher/run-ephemeral-host-publisher-gate.ps1 | 7 +++++++ tests/publisher/sandbox-conformance.ps1 | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/publisher/run-ephemeral-host-publisher-gate.ps1 b/tests/publisher/run-ephemeral-host-publisher-gate.ps1 index 7506a1d..8077f89 100644 --- a/tests/publisher/run-ephemeral-host-publisher-gate.ps1 +++ b/tests/publisher/run-ephemeral-host-publisher-gate.ps1 @@ -42,6 +42,12 @@ $wingetCommand = Get-Command winget.exe -ErrorAction SilentlyContinue if ($null -eq $wingetCommand) { throw 'The GitHub-hosted Windows runner does not expose the stock winget.exe execution alias.' } +$clientVersionOutput = & $wingetCommand.Source --version 2>&1 +if ($LASTEXITCODE -ne 0 -or ($clientVersionOutput -join '').Trim() -notmatch '^v(\d+\.\d+\.\d+)$') { + throw "Could not identify the stock WinGet client version: $($clientVersionOutput -join ' ')" +} +$clientVersion = $Matches[1] +Write-Output "Stock App Installer package $($appInstaller.Version); WinGet client $clientVersion." $workRoot = Join-Path $env:RUNNER_TEMP "pinget-stock-gate-$([Guid]::NewGuid().ToString('N'))" $signerPfxPath = Join-Path $workRoot 'publisher-signing.pfx' @@ -272,6 +278,7 @@ try { -ExecutionTarget EphemeralHost ` -AllowEphemeralHostMutations ` -ExpectedWinGetVersion $appInstaller.Version.ToString() ` + -ExpectedWinGetClientVersion $clientVersion ` -SignerCertificatePath $signerCertificatePath ` -PublisherCliPath $publisher.Cli ` -PublisherStateDirectory $stateDirectory ` diff --git a/tests/publisher/sandbox-conformance.ps1 b/tests/publisher/sandbox-conformance.ps1 index 4b56667..4a27b79 100644 --- a/tests/publisher/sandbox-conformance.ps1 +++ b/tests/publisher/sandbox-conformance.ps1 @@ -8,6 +8,7 @@ param( [string] $ExecutionTarget = 'EphemeralHost', [switch] $AllowEphemeralHostMutations, [string] $ExpectedWinGetVersion = '1.29.380.0', + [string] $ExpectedWinGetClientVersion, [string] $PublisherCliPath, [string] $PublisherStateDirectory, [string] $PublisherInputDirectory, @@ -643,7 +644,11 @@ try { $script:winget = $command.Source } $versionOutput = & $script:winget --version 2>&1 - $expectedClientVersion = $ExpectedWinGetVersion -replace '\.0$', '' + $expectedClientVersion = if ([string]::IsNullOrWhiteSpace($ExpectedWinGetClientVersion)) { + $ExpectedWinGetVersion -replace '\.0$', '' + } else { + $ExpectedWinGetClientVersion + } if ($LASTEXITCODE -ne 0 -or ($versionOutput -join '') -notmatch [regex]::Escape($expectedClientVersion)) { throw "Unexpected WinGet client version: $($versionOutput -join ' ')" } From 30597f24fe3c97edfdbe43d7ad9abf0ba92e3f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sat, 31 Oct 2026 22:20:07 -0400 Subject: [PATCH 7/8] Record hardened signed WinGet acceptance on hosted Windows CI Document successful stock WinGet 1.11.510 interoperability for both publishers and report the CLI version separately from App Installer in the gate summary. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/repository-publishing.md | 10 ++++++++-- tests/publisher/run-ephemeral-host-publisher-gate.ps1 | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/repository-publishing.md b/docs/repository-publishing.md index ed4d7c0..f459a02 100644 --- a/docs/repository-publishing.md +++ b/docs/repository-publishing.md @@ -189,8 +189,14 @@ readers consumed both publishers' 500-version feeds and selected `1.0.499`, exercising multi-block MSZIP sidecars end to end. These results establish the tested client/toolchain baseline, not universal compatibility with older clients or every manifest field. Current artifact-format-v2 outputs passed -the local exact-byte comparison and independent SQLite/MSZIP checks, but -have not received a fresh signed stock-client acceptance run. +the local exact-byte comparison and independent SQLite/MSZIP checks. +Hardened signed outputs from both publishers also passed stock WinGet +**1.11.510** (App Installer **1.26.510.0**) on a disposable GitHub-hosted +Windows runner. That gate covers source add/update, search/show, multi-block +version data, hash-verified download, version bumps, package additions/removals +and cached-index metadata retention. It generates short-lived signing and TLS +credentials under runner temporary storage, then verifies temporary source and +certificate cleanup. No Sandbox or developer-host trust changes are needed. Failure tests cover packaging/signing errors, update promotion rollback and destination locking. The .NET publisher also has an end-to-end regression diff --git a/tests/publisher/run-ephemeral-host-publisher-gate.ps1 b/tests/publisher/run-ephemeral-host-publisher-gate.ps1 index 8077f89..8ce0e5a 100644 --- a/tests/publisher/run-ephemeral-host-publisher-gate.ps1 +++ b/tests/publisher/run-ephemeral-host-publisher-gate.ps1 @@ -304,7 +304,7 @@ try { } $gateSucceeded = $true - Write-Output "PASS signed stock WinGet conformance for Rust and .NET on GitHub runner with WinGet $($appInstaller.Version)." + Write-Output "PASS signed stock WinGet conformance for Rust and .NET on GitHub runner with WinGet $clientVersion (App Installer $($appInstaller.Version))." } finally { $cleanupErrors = [System.Collections.Generic.List[string]]::new() From 363e187e25b0c0df29214586842555e396d9d455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Sun, 1 Nov 2026 09:01:20 -0500 Subject: [PATCH 8/8] Replace Python publisher conformance helpers with PowerShell Preserve independent native SQLite and MSZIP validation, fixture generation and exact cross-language artifact comparison using existing build dependencies. Remove Python and PyYAML provisioning from CI. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 8 - tests/publisher/build_v2_fixture.ps1 | 203 ++++++++ tests/publisher/build_v2_fixture.py | 392 ---------------- .../compare_production_snapshots.ps1 | 233 ++++++++++ .../publisher/compare_production_snapshots.py | 258 ----------- tests/publisher/contract.md | 14 +- tests/publisher/publisher-validation.ps1 | 437 ++++++++++++++++++ .../publisher/run-production-conformance.ps1 | 81 +++- .../validate_production_snapshot.ps1 | 161 +++++++ .../publisher/validate_production_snapshot.py | 202 -------- 10 files changed, 1110 insertions(+), 879 deletions(-) create mode 100644 tests/publisher/build_v2_fixture.ps1 delete mode 100644 tests/publisher/build_v2_fixture.py create mode 100644 tests/publisher/compare_production_snapshots.ps1 delete mode 100644 tests/publisher/compare_production_snapshots.py create mode 100644 tests/publisher/publisher-validation.ps1 create mode 100644 tests/publisher/validate_production_snapshot.ps1 delete mode 100644 tests/publisher/validate_production_snapshot.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbfd4e5..2f2c561 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,14 +116,6 @@ jobs: - name: Run publisher tests run: dotnet test dotnet/src/Devolutions.Pinget.Publisher.Tests/Devolutions.Pinget.Publisher.Tests.csproj -c Release - - name: Setup Python for independent publisher validation - uses: actions/setup-python@v6 - with: - python-version: '3.13' - - - name: Install independent publisher validator dependency - run: python -m pip install "PyYAML>=6,<7" - - name: Run Rust and .NET publisher interoperability shell: pwsh run: | diff --git a/tests/publisher/build_v2_fixture.ps1 b/tests/publisher/build_v2_fixture.ps1 new file mode 100644 index 0000000..5a84399 --- /dev/null +++ b/tests/publisher/build_v2_fixture.ps1 @@ -0,0 +1,203 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $Output, + [Parameter(Mandatory = $true)][string] $MakeAppx, + [string] $Publisher = 'CN=Pinget Publisher Contract Test', + [string] $InstallerUrl = 'https://127.0.0.1:8765/HarmlessPayload.txt', + [string] $SqliteLibraryPath = '' +) + +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'publisher-validation.ps1') + +if (-not [System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform( + [System.Runtime.InteropServices.OSPlatform]::Windows)) { + throw 'The WinGet v2 fixture requires the native Windows Compression API.' +} +if ([string]::IsNullOrWhiteSpace($SqliteLibraryPath)) { + $runtime = switch ([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture) { + 'X64' { 'win-x64' } + 'Arm64' { 'win-arm64' } + 'X86' { 'win-x86' } + default { throw "Unsupported architecture for native SQLite inspection: $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture)" } + } + $SqliteLibraryPath = Join-Path $PSScriptRoot "..\..\dotnet\src\Devolutions.Pinget.Cli\bin\Release\net10.0\runtimes\$runtime\native\e_sqlite3.dll" +} +Initialize-PublisherValidation -SqliteLibraryPath $SqliteLibraryPath + +$outputPath = [System.IO.Path]::GetFullPath($Output) +if (Test-Path -LiteralPath $outputPath) { + throw "Refusing to overwrite fixture output directory: $outputPath" +} +if (-not (Test-Path -LiteralPath $MakeAppx -PathType Leaf)) { + throw "MakeAppx does not exist: $MakeAppx" +} +$fixtureRoot = Join-Path $PSScriptRoot 'fixtures' +$feedPath = Join-Path $outputPath 'feed' +$stagePath = Join-Path $outputPath 'package-staging' +$manifestRelative = 'manifests/Pinget.Contract.Singleton.yaml' +$sidecarPrefix = 'packages/Pinget.Contract.Singleton' +$payload = [System.Text.Encoding]::UTF8.GetBytes("Harmless Pinget contract fixture. Download only; never execute.`n") +$payloadHash = Get-Sha256Hex $payload +$installerUrl = $InstallerUrl.Replace("'", "''") +if ($installerUrl.Contains("`n") -or $installerUrl.Contains("`r")) { + throw 'Installer URL must be a single line.' +} + +New-Item -ItemType Directory -Path (Join-Path $feedPath 'manifests') -Force | Out-Null +New-Item -ItemType Directory -Path (Join-Path $feedPath 'packages') -Force | Out-Null +New-Item -ItemType Directory -Path (Join-Path $stagePath 'Public') -Force | Out-Null +New-Item -ItemType Directory -Path (Join-Path $stagePath 'Assets') -Force | Out-Null + +$payloadPath = Join-Path $feedPath 'HarmlessPayload.txt' +[System.IO.File]::WriteAllBytes($payloadPath, $payload) +$manifestTemplate = Get-Content -LiteralPath (Join-Path $fixtureRoot 'singleton.yaml.in') -Raw +$manifestText = $manifestTemplate.Replace('{{INSTALLER_URL}}', $installerUrl).Replace('{{INSTALLER_SHA256}}', $payloadHash) +$manifestText = $manifestText -replace "`r`n|`r", "`n" +$manifestBytes = [System.Text.UTF8Encoding]::new($false).GetBytes($manifestText) +[System.IO.File]::WriteAllBytes((Join-Path $feedPath $manifestRelative.Replace('/', '\')), $manifestBytes) + +$manifestHash = Get-Sha256Hex $manifestBytes +$versionDataText = "sV: `"1.0`"`nvD:`n - v: `"1.0.0`"`n rP: $manifestRelative`n s256H: $manifestHash`n" +$versionData = [System.Text.UTF8Encoding]::new($false).GetBytes($versionDataText) +$compressedSidecar = [PingetPublisher.Validation.NativeMsZip]::Compress($versionData) +$sidecarHash = Get-Sha256Hex $compressedSidecar +$sidecarRelative = "$sidecarPrefix/$($sidecarHash.Substring(0, 8))/versionData.mszyml" +$sidecarPath = Join-Path $feedPath $sidecarRelative.Replace('/', '\') +New-Item -ItemType Directory -Path (Split-Path -Parent $sidecarPath) -Force | Out-Null +[System.IO.File]::WriteAllBytes($sidecarPath, $compressedSidecar) +[System.IO.File]::WriteAllBytes((Join-Path $outputPath 'versionData.yml'), $versionData) + +$databasePath = Join-Path $stagePath 'Public\index.db' +$schema = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'index-schema.sql') -Raw +$databaseIdentifier = '{' + [Guid]::NewGuid().ToString().ToUpperInvariant() + '}' +$sidecarHashSql = $sidecarHash +$sql = @" +$schema +INSERT INTO metadata(name, value) VALUES + ('majorVersion', '2'), + ('minorVersion', '0'), + ('databaseIdentifier', '$databaseIdentifier'), + ('lastwritetime', '1793380000'); +INSERT INTO packages(rowid, id, name, moniker, latest_version, arp_min_version, arp_max_version, hash) +VALUES(1, 'Pinget.Contract.Singleton', 'Pinget Contract Fixture', 'pinget-contract', '1.0.0', NULL, NULL, X'$sidecarHashSql'); +INSERT INTO tags2(rowid, tag) VALUES(1, 'publisher-contract'); +INSERT INTO tags2_map(tag, package) VALUES(1, 1); +"@ +[PingetPublisher.Validation.NativeSqlite]::Execute($databasePath, $sql) + +$appxTemplate = Get-Content -LiteralPath (Join-Path $fixtureRoot 'AppxManifest.xml.in') -Raw +$escapedPublisher = [System.Security.SecurityElement]::Escape($Publisher) +$appxManifest = $appxTemplate.Replace('{{PUBLISHER}}', $escapedPublisher) -replace "`r`n|`r", "`n" +[System.IO.File]::WriteAllText( + (Join-Path $stagePath 'AppxManifest.xml'), + $appxManifest, + [System.Text.UTF8Encoding]::new($false) +) +$logo = [Convert]::FromHexString( + '89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000b49444154789c636000020000050001a5f645400000000049454e44ae426082' +) +foreach ($name in @('StoreLogo.png', 'Logo150.png', 'Logo44.png')) { + [System.IO.File]::WriteAllBytes((Join-Path $stagePath "Assets\$name"), $logo) +} + +$packagePath = Join-Path $feedPath 'source2.msix' +$packOutput = & $MakeAppx pack /d $stagePath /p $packagePath /o 2>&1 +if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $packagePath -PathType Leaf)) { + throw "MakeAppx failed to package the v2 fixture (exit $LASTEXITCODE): $($packOutput -join "`n")" +} + +$metadataText = @" +Package=source2.msix +Index=Public/index.db +Manifest=$manifestRelative +Sidecar=$sidecarRelative +SidecarSHA256=$sidecarHash +InstallerURL=$InstallerUrl +InstallerSHA256=$payloadHash +Publisher=$Publisher +"@ +[System.IO.File]::WriteAllText( + (Join-Path $outputPath 'fixture-metadata.txt'), + $metadataText, + [System.Text.UTF8Encoding]::new($false) +) + +# Independent fixture self-checks: SQLite schema/index, exact native MSZIP bytes, +# and the packaged MSIX/index/manifest/resource contract. +$integrity = [PingetPublisher.Validation.NativeSqlite]::Query($databasePath, 'PRAGMA integrity_check') +if ($integrity.Count -ne 1 -or $integrity[0].integrity_check -ne 'ok') { + throw "Fixture SQLite integrity_check failed: $($integrity | ConvertTo-Json -Compress)" +} +$expectedTables = @('commands2', 'commands2_map', 'metadata', 'norm_names2', 'norm_publishers2', 'packages', 'pfns2', 'productcodes2', 'tags2', 'tags2_map', 'upgradecodes2') +Assert-Equal (Get-SqliteTableNames $databasePath) $expectedTables 'Fixture WinGet v2 table set differs.' +$metadataRows = [PingetPublisher.Validation.NativeSqlite]::Query($databasePath, 'SELECT name, value FROM metadata') +$metadataMap = @{} +foreach ($row in $metadataRows) { $metadataMap[[string]$row.name] = [string]$row.value } +if ($metadataMap.majorVersion -ne '2' -or $metadataMap.minorVersion -ne '0' -or + $metadataMap.databaseIdentifier -notmatch '^\{[0-9A-F-]{36}\}$' -or + $metadataMap.lastwritetime -notmatch '^\d+$') { + throw "Fixture SQLite metadata is invalid: $($metadataMap | ConvertTo-Json -Compress)" +} +$package = [PingetPublisher.Validation.NativeSqlite]::Query($databasePath, "SELECT id, name, latest_version, hash FROM packages WHERE rowid=1") +if ($package.Count -ne 1 -or $package[0].id -ne 'Pinget.Contract.Singleton' -or + $package[0].name -ne 'Pinget Contract Fixture' -or $package[0].latest_version -ne '1.0.0' -or + $package[0].hash.Length -ne 32) { + throw 'Fixture SQLite package row or sidecar digest is invalid.' +} +$tagMapping = [PingetPublisher.Validation.NativeSqlite]::Query($databasePath, 'SELECT tag, package FROM tags2_map') +if ($tagMapping.Count -ne 1 -or $tagMapping[0].tag -ne 1 -or $tagMapping[0].package -ne 1) { + throw 'Fixture SQLite tag mapping is invalid.' +} +$decodedVersionData = [PingetPublisher.Validation.NativeMsZip]::Decompress([System.IO.File]::ReadAllBytes($sidecarPath)) +if (-not (Test-ByteArrayEqual $versionData $decodedVersionData) -or + (Get-Sha256Hex ([System.IO.File]::ReadAllBytes($sidecarPath))) -ne $sidecarHash -or + -not (Test-ByteArrayEqual $package[0].hash ([Convert]::FromHexString($sidecarHash)))) { + throw 'Fixture sidecar bytes, MSZIP round trip, or index hash do not match.' +} +if ($sidecarRelative -ne "packages/Pinget.Contract.Singleton/$($sidecarHash.Substring(0, 8).ToLowerInvariant())/versionData.mszyml") { + throw 'Fixture sidecar path does not use the lowercase digest prefix.' +} +Write-Output 'PASS Fixture_SqliteV2SchemaAndPackageMetadata' +Write-Output 'PASS Fixture_NativeMsZipAndSidecarHashParity' + +$archive = [System.IO.Compression.ZipFile]::OpenRead($packagePath) +try { + $entryNames = @($archive.Entries | ForEach-Object { $_.FullName }) + foreach ($requiredEntry in @('Public/index.db', 'AppxManifest.xml')) { + if ($entryNames -notcontains $requiredEntry) { throw "Fixture MSIX is missing $requiredEntry." } + } + $indexEntry = $archive.GetEntry('Public/index.db') + $indexStream = [System.IO.MemoryStream]::new() + $entryStream = $indexEntry.Open() + try { $entryStream.CopyTo($indexStream) } finally { $entryStream.Dispose() } + if (-not (Test-ByteArrayEqual $indexStream.ToArray() ([System.IO.File]::ReadAllBytes($databasePath)))) { + throw 'Packaged Public/index.db differs from the independently inspected staged database.' + } + $manifestEntry = $archive.GetEntry('AppxManifest.xml') + $reader = [System.IO.StreamReader]::new($manifestEntry.Open()) + try { [xml]$packageXml = $reader.ReadToEnd() } finally { $reader.Dispose() } + $namespace = [System.Xml.XmlNamespaceManager]::new($packageXml.NameTable) + $namespace.AddNamespace('uap3', 'http://schemas.microsoft.com/appx/manifest/uap/windows10/3') + $extension = $packageXml.SelectSingleNode('//uap3:AppExtension', $namespace) + if ($null -eq $extension -or $extension.GetAttribute('Name') -ne 'com.microsoft.winget.source' -or + $extension.GetAttribute('PublicFolder') -ne 'Public') { + throw 'Packaged MSIX does not declare the WinGet source extension and Public folder.' + } + if ($entryNames -contains $manifestRelative -or $entryNames -contains $sidecarRelative) { + throw 'Hosted manifest or sidecar was incorrectly embedded in the MSIX.' + } + foreach ($externalArtifact in @($manifestRelative, $sidecarRelative)) { + if (-not (Test-Path -LiteralPath (Join-Path $feedPath $externalArtifact.Replace('/', '\')) -PathType Leaf)) { + throw "Fixture hosted artifact is missing: $externalArtifact" + } + } + if ((Get-FileSha256Hex $payloadPath) -ne $payloadHash) { + throw 'Fixture harmless payload SHA-256 does not match the recorded metadata.' + } +} +finally { + $archive.Dispose() +} +Write-Output 'PASS Fixture_MsixContainsV2IndexAndHostedArtifactLayout' +Write-Output "Fixture output: $outputPath" diff --git a/tests/publisher/build_v2_fixture.py b/tests/publisher/build_v2_fixture.py deleted file mode 100644 index 40f0595..0000000 --- a/tests/publisher/build_v2_fixture.py +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env python3 -"""Build and self-check a locally authored WinGet schema 2.0 source fixture.""" - -from __future__ import annotations - -import argparse -import ctypes -import hashlib -import sqlite3 -import subprocess -import sys -import unittest -import uuid -import zipfile -from pathlib import Path -from xml.etree import ElementTree - - -ROOT = Path(__file__).resolve().parent -FIXTURES = ROOT / "fixtures" -SCHEMA = ROOT / "index-schema.sql" -PAYLOAD = b"Harmless Pinget contract fixture. Download only; never execute.\n" -MANIFEST_RELATIVE = "manifests/Pinget.Contract.Singleton.yaml" -SIDECAR_RELATIVE = ( - "packages/Pinget.Contract.Singleton/{hash_prefix}/versionData.mszyml" -) -HASH_RE = set("0123456789abcdef") -ARTIFACTS: Path | None = None - - -def sha256(data: bytes) -> bytes: - return hashlib.sha256(data).digest() - - -def compress_mszip(data: bytes) -> bytes: - """Use the same native Windows Compression API as WinGet's producer.""" - api = ctypes.WinDLL("cabinet.dll", use_last_error=True) - handle = ctypes.c_void_p() - api.CreateCompressor.argtypes = [ - ctypes.c_uint32, - ctypes.c_void_p, - ctypes.POINTER(ctypes.c_void_p), - ] - api.CreateCompressor.restype = ctypes.c_int - api.Compress.argtypes = [ - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.POINTER(ctypes.c_size_t), - ] - api.Compress.restype = ctypes.c_int - api.CloseCompressor.argtypes = [ctypes.c_void_p] - api.CloseCompressor.restype = ctypes.c_int - - if not api.CreateCompressor(2, None, ctypes.byref(handle)): - raise ctypes.WinError(ctypes.get_last_error()) - try: - source = ctypes.create_string_buffer(data) - required = ctypes.c_size_t() - api.Compress( - handle, source, len(data), None, 0, ctypes.byref(required) - ) - if ctypes.get_last_error() != 122: - raise ctypes.WinError(ctypes.get_last_error()) - output = ctypes.create_string_buffer(required.value) - actual = ctypes.c_size_t() - if not api.Compress( - handle, - source, - len(data), - output, - required.value, - ctypes.byref(actual), - ): - raise ctypes.WinError(ctypes.get_last_error()) - return output.raw[: actual.value] - finally: - api.CloseCompressor(handle) - - -def decompress_mszip(data: bytes) -> bytes: - """Independently verify the stored stream using the native decompressor.""" - api = ctypes.WinDLL("cabinet.dll", use_last_error=True) - handle = ctypes.c_void_p() - api.CreateDecompressor.argtypes = [ - ctypes.c_uint32, - ctypes.c_void_p, - ctypes.POINTER(ctypes.c_void_p), - ] - api.CreateDecompressor.restype = ctypes.c_int - api.Decompress.argtypes = [ - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.POINTER(ctypes.c_size_t), - ] - api.Decompress.restype = ctypes.c_int - api.CloseDecompressor.argtypes = [ctypes.c_void_p] - api.CloseDecompressor.restype = ctypes.c_int - - if not api.CreateDecompressor(2, None, ctypes.byref(handle)): - raise ctypes.WinError(ctypes.get_last_error()) - try: - source = ctypes.create_string_buffer(data) - required = ctypes.c_size_t() - api.Decompress( - handle, source, len(data), None, 0, ctypes.byref(required) - ) - if ctypes.get_last_error() != 122: - raise ctypes.WinError(ctypes.get_last_error()) - output = ctypes.create_string_buffer(required.value) - actual = ctypes.c_size_t() - if not api.Decompress( - handle, - source, - len(data), - output, - required.value, - ctypes.byref(actual), - ): - raise ctypes.WinError(ctypes.get_last_error()) - return output.raw[: actual.value] - finally: - api.CloseDecompressor(handle) - - -def create_database(path: Path, compressed_sidecar: bytes) -> None: - connection = sqlite3.connect(path) - try: - connection.executescript(SCHEMA.read_text(encoding="utf-8")) - with connection: - connection.executemany( - "INSERT INTO metadata(name, value) VALUES(?, ?)", - [ - ("majorVersion", "2"), - ("minorVersion", "0"), - ("databaseIdentifier", "{" + str(uuid.uuid4()).upper() + "}"), - ("lastwritetime", "1793380000"), - ], - ) - connection.execute( - """INSERT INTO packages( - rowid, id, name, moniker, latest_version, - arp_min_version, arp_max_version, hash - ) VALUES(1, ?, ?, ?, ?, NULL, NULL, ?)""", - ( - "Pinget.Contract.Singleton", - "Pinget Contract Fixture", - "pinget-contract", - "1.0.0", - sqlite3.Binary(sha256(compressed_sidecar)), - ), - ) - connection.execute( - "INSERT INTO tags2(rowid, tag) VALUES(1, 'publisher-contract')" - ) - connection.execute( - "INSERT INTO tags2_map(tag, package) VALUES(1, 1)" - ) - finally: - connection.close() - - -def build(output: Path, makeappx: Path, publisher: str, installer_url: str) -> None: - global ARTIFACTS - ARTIFACTS = output.resolve() - if ARTIFACTS.exists(): - raise FileExistsError( - f"Refusing to overwrite fixture output directory: {ARTIFACTS}" - ) - feed = ARTIFACTS / "feed" - stage = ARTIFACTS / "package-staging" - (feed / "manifests").mkdir(parents=True) - (feed / "packages").mkdir() - (stage / "Public").mkdir(parents=True) - (stage / "Assets").mkdir() - - payload_path = feed / "HarmlessPayload.txt" - payload_path.write_bytes(PAYLOAD) - manifest_template = (FIXTURES / "singleton.yaml.in").read_text( - encoding="utf-8" - ) - manifest = manifest_template.replace("{{INSTALLER_URL}}", installer_url) - manifest = manifest.replace( - "{{INSTALLER_SHA256}}", hashlib.sha256(PAYLOAD).hexdigest() - ) - manifest_bytes = manifest.encode("utf-8") - (feed / MANIFEST_RELATIVE).write_bytes(manifest_bytes) - - version_data = ( - 'sV: "1.0"\n' - "vD:\n" - ' - v: "1.0.0"\n' - f" rP: {MANIFEST_RELATIVE}\n" - f" s256H: {hashlib.sha256(manifest_bytes).hexdigest()}\n" - ).encode("utf-8") - compressed_sidecar = compress_mszip(version_data) - sidecar_hash = hashlib.sha256(compressed_sidecar).hexdigest() - sidecar_relative = SIDECAR_RELATIVE.format(hash_prefix=sidecar_hash[:8]) - sidecar_path = feed / Path(sidecar_relative) - sidecar_path.parent.mkdir(parents=True) - sidecar_path.write_bytes(compressed_sidecar) - (ARTIFACTS / "versionData.yml").write_bytes(version_data) - - create_database(stage / "Public" / "index.db", compressed_sidecar) - - appx_template = (FIXTURES / "AppxManifest.xml.in").read_text( - encoding="utf-8" - ) - appx_manifest = appx_template.replace("{{PUBLISHER}}", publisher) - (stage / "AppxManifest.xml").write_text( - appx_manifest, encoding="utf-8", newline="\n" - ) - # Locally-authored, one-pixel transparent PNG used only as required package assets. - logo = bytes.fromhex( - "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489" - "0000000b49444154789c636000020000050001a5f645400000000049454e44ae426082" - ) - for name in ("StoreLogo.png", "Logo150.png", "Logo44.png"): - (stage / "Assets" / name).write_bytes(logo) - - package_path = feed / "source2.msix" - subprocess.run( - [str(makeappx), "pack", "/d", str(stage), "/p", str(package_path), "/o"], - check=True, - ) - (ARTIFACTS / "fixture-metadata.txt").write_text( - f"Package=source2.msix\n" - f"Index=Public/index.db\n" - f"Manifest={MANIFEST_RELATIVE}\n" - f"Sidecar={sidecar_relative}\n" - f"SidecarSHA256={sidecar_hash}\n" - f"InstallerURL={installer_url}\n" - f"InstallerSHA256={hashlib.sha256(PAYLOAD).hexdigest()}\n" - f"Publisher={publisher}\n", - encoding="utf-8", - ) - - -class V2FixtureTests(unittest.TestCase): - def setUp(self) -> None: - if ARTIFACTS is None: - self.fail("fixture must be built before running contract checks") - - def test_sqlite_v2_schema_and_package_metadata(self) -> None: - assert ARTIFACTS is not None - path = ARTIFACTS / "package-staging" / "Public" / "index.db" - with sqlite3.connect(path) as database: - self.assertEqual("ok", database.execute("PRAGMA integrity_check").fetchone()[0]) - tables = { - row[0] - for row in database.execute( - "SELECT name FROM sqlite_master WHERE type='table'" - ) - } - self.assertEqual( - { - "metadata", - "packages", - "tags2", - "tags2_map", - "commands2", - "commands2_map", - "pfns2", - "productcodes2", - "norm_names2", - "norm_publishers2", - "upgradecodes2", - }, - tables, - ) - metadata = dict(database.execute("SELECT name, value FROM metadata")) - self.assertEqual("2", metadata["majorVersion"]) - self.assertEqual("0", metadata["minorVersion"]) - self.assertRegex(metadata["databaseIdentifier"], r"^\{[0-9A-F-]{36}\}$") - self.assertTrue(metadata["lastwritetime"].isdigit()) - package = database.execute( - "SELECT id, name, latest_version, hash FROM packages WHERE rowid=1" - ).fetchone() - self.assertEqual( - ( - "Pinget.Contract.Singleton", - "Pinget Contract Fixture", - "1.0.0", - ), - package[:3], - ) - self.assertEqual(32, len(package[3])) - self.assertEqual( - [(1, 1)], - database.execute( - "SELECT tag, package FROM tags2_map" - ).fetchall(), - ) - - def test_sidecar_hash_matches_native_mszip_stream(self) -> None: - assert ARTIFACTS is not None - metadata = dict( - line.split("=", 1) - for line in (ARTIFACTS / "fixture-metadata.txt") - .read_text(encoding="utf-8") - .splitlines() - ) - package_hash = bytes.fromhex(metadata["SidecarSHA256"]) - expected_sidecar = ( - "packages/Pinget.Contract.Singleton/" - f"{metadata['SidecarSHA256'][:8].lower()}/versionData.mszyml" - ) - self.assertEqual(expected_sidecar, metadata["Sidecar"]) - sidecar = ( - ARTIFACTS - / "feed" - / Path(metadata["Sidecar"].replace("/", "\\")) - ).read_bytes() - self.assertEqual(package_hash, sha256(sidecar)) - self.assertEqual( - (ARTIFACTS / "versionData.yml").read_bytes(), - decompress_mszip(sidecar), - ) - with sqlite3.connect( - ARTIFACTS / "package-staging" / "Public" / "index.db" - ) as database: - index_hash = database.execute( - "SELECT hash FROM packages WHERE id=?", - ("Pinget.Contract.Singleton",), - ).fetchone()[0] - self.assertEqual(package_hash, index_hash) - self.assertTrue(set(metadata["SidecarSHA256"]) <= HASH_RE) - - def test_msix_contains_the_v2_index_and_hosted_artifacts(self) -> None: - assert ARTIFACTS is not None - package_path = ARTIFACTS / "feed" / "source2.msix" - with zipfile.ZipFile(package_path) as package: - names = set(package.namelist()) - self.assertIn("Public/index.db", names) - self.assertIn("AppxManifest.xml", names) - packaged_index = package.read("Public/index.db") - staged_index = ( - ARTIFACTS / "package-staging" / "Public" / "index.db" - ).read_bytes() - self.assertEqual(staged_index, packaged_index) - package_xml = ElementTree.fromstring( - package.read("AppxManifest.xml") - ) - app_extension = next( - node - for node in package_xml.iter() - if node.tag.endswith("AppExtension") - ) - self.assertEqual("com.microsoft.winget.source", app_extension.attrib["Name"]) - self.assertEqual("Public", app_extension.attrib["PublicFolder"]) - metadata = dict( - line.split("=", 1) - for line in (ARTIFACTS / "fixture-metadata.txt") - .read_text(encoding="utf-8") - .splitlines() - ) - self.assertNotIn(metadata["Manifest"], names) - self.assertNotIn(metadata["Sidecar"], names) - self.assertTrue((ARTIFACTS / "feed" / metadata["Manifest"]).is_file()) - self.assertTrue((ARTIFACTS / "feed" / metadata["Sidecar"]).is_file()) - self.assertEqual( - hashlib.sha256(PAYLOAD).hexdigest(), - metadata["InstallerSHA256"], - ) - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--makeappx", type=Path, required=True) - parser.add_argument( - "--publisher", default="CN=Pinget Publisher Contract Test" - ) - parser.add_argument( - "--installer-url", - default="https://127.0.0.1:8765/HarmlessPayload.txt", - ) - args = parser.parse_args() - build(args.output, args.makeappx, args.publisher, args.installer_url) - result = unittest.TextTestRunner(verbosity=2).run( - unittest.defaultTestLoader.loadTestsFromTestCase(V2FixtureTests) - ) - return 0 if result.wasSuccessful() else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/publisher/compare_production_snapshots.ps1 b/tests/publisher/compare_production_snapshots.ps1 new file mode 100644 index 0000000..f76e960 --- /dev/null +++ b/tests/publisher/compare_production_snapshots.ps1 @@ -0,0 +1,233 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $RustSnapshot, + [Parameter(Mandatory = $true)][string] $DotnetSnapshot, + [Parameter(Mandatory = $true)][string] $SqliteLibraryPath, + [Parameter(Mandatory = $true)][string] $YamlDotNetPath +) + +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'publisher-validation.ps1') +Initialize-PublisherValidation -SqliteLibraryPath $SqliteLibraryPath -YamlDotNetPath $YamlDotNetPath + +function Get-SnapshotMetadata { + param([string] $SnapshotPath) + $snapshot = [System.IO.Path]::GetFullPath((Resolve-Path -LiteralPath $SnapshotPath).Path) + $metadataPath = Join-Path $snapshot 'snapshot.json' + if (-not (Test-Path -LiteralPath $metadataPath -PathType Leaf)) { + throw "Snapshot metadata is missing: $metadataPath" + } + return [pscustomobject]@{ + Path = $snapshot + Metadata = Get-Content -LiteralPath $metadataPath -Raw | ConvertFrom-Json -AsHashtable + } +} + +function Get-MappedValues { + param( + [string] $DatabasePath, [string] $PackageId, + [string] $ValueTable, [string] $ValueColumn, + [string] $MapTable, [string] $MapColumn + ) + $query = "SELECT valuesTable.$ValueColumn AS value FROM $ValueTable AS valuesTable " + + "JOIN $MapTable AS valueMap ON valueMap.$MapColumn=valuesTable.rowid " + + 'JOIN packages AS package ON package.rowid=valueMap.package ' + + "WHERE package.id='$($PackageId.Replace("'", "''"))' ORDER BY valuesTable.$ValueColumn" + return @([PingetPublisher.Validation.NativeSqlite]::Query($DatabasePath, $query) | + ForEach-Object { [string]$_.value }) +} + +function Get-CorrelationValues { + param([string] $DatabasePath, [string] $PackageId, [string] $Table, [string] $Column) + $query = "SELECT value.$Column AS value FROM $Table AS value " + + "JOIN packages AS package ON package.rowid=value.package " + + "WHERE package.id='$($PackageId.Replace("'", "''"))' ORDER BY value.$Column" + return @([PingetPublisher.Validation.NativeSqlite]::Query($DatabasePath, $query) | + ForEach-Object { [string]$_.value }) +} + +function Read-LogicalIndex { + param([string] $SnapshotPath) + $databasePath = Join-Path $SnapshotPath 'Public\index.db' + $rows = [PingetPublisher.Validation.NativeSqlite]::Query( + $databasePath, + 'SELECT id, name, moniker, latest_version, arp_min_version, arp_max_version FROM packages' + ) + $packages = [System.Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal) + foreach ($row in $rows) { + $id = [string]$row.id + $packages.Add($id, @{ + name = $row.name + moniker = $row.moniker + latestVersion = $row.latest_version + arpMinimum = $row.arp_min_version + arpMaximum = $row.arp_max_version + tags = @(Get-MappedValues $databasePath $id 'tags2' 'tag' 'tags2_map' 'tag') + commands = @(Get-MappedValues $databasePath $id 'commands2' 'command' 'commands2_map' 'command') + pfns = @(Get-CorrelationValues $databasePath $id 'pfns2' 'pfn') + productCodes = @(Get-CorrelationValues $databasePath $id 'productcodes2' 'productcode') + upgradeCodes = @(Get-CorrelationValues $databasePath $id 'upgradecodes2' 'upgradecode') + normalizedNames = @(Get-CorrelationValues $databasePath $id 'norm_names2' 'norm_name') + normalizedPublishers = @(Get-CorrelationValues $databasePath $id 'norm_publishers2' 'norm_publisher') + }) + } + return $packages +} + +function Read-CompressedSidecars { + param([string] $SnapshotPath) + $databasePath = Join-Path $SnapshotPath 'Public\index.db' + $feedPath = Join-Path $SnapshotPath 'feed' + $rows = [PingetPublisher.Validation.NativeSqlite]::Query($databasePath, 'SELECT id, hash FROM packages') + $sidecars = [System.Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal) + foreach ($row in $rows) { + $packageId = [string]$row.id + if ($row.hash -isnot [byte[]] -or $row.hash.Length -ne 32) { + throw "Package $packageId has an invalid sidecar hash in the SQLite index." + } + $sidecar = Get-PackageSidecar -FeedPath $feedPath -PackageId $packageId -HashBytes $row.hash + $bytes = [System.IO.File]::ReadAllBytes($sidecar.Path) + if ((Get-Sha256Hex $bytes) -cne $sidecar.Hash) { + throw "Package $packageId compressed sidecar hash does not match the SQLite index." + } + $sidecars.Add($packageId, [pscustomobject]@{ Hash = $sidecar.Hash; Bytes = $bytes }) + } + return $sidecars +} + +function Read-VersionDocuments { + param([string] $SnapshotPath) + $databasePath = Join-Path $SnapshotPath 'Public\index.db' + $feedPath = Join-Path $SnapshotPath 'feed' + $rows = [PingetPublisher.Validation.NativeSqlite]::Query($databasePath, 'SELECT id, hash FROM packages') + $versions = [System.Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal) + foreach ($row in $rows) { + $sidecar = Get-PackageSidecar -FeedPath $feedPath -PackageId ([string]$row.id) -HashBytes $row.hash + $raw = [PingetPublisher.Validation.NativeMsZip]::Decompress([System.IO.File]::ReadAllBytes($sidecar.Path)) + $text = [System.Text.UTF8Encoding]::new($false, $true).GetString($raw) + $docs = ConvertFrom-YamlDocuments $text + if ($docs.Count -ne 1 -or $docs[0] -isnot [System.Collections.IDictionary]) { + throw "Package $($row.id) version-data stream is not one mapping document." + } + $versions.Add([string]$row.id, $docs[0]) + } + return $versions +} + +function Get-ManifestIdentity { + param([System.Collections.IDictionary] $Manifest) + return (@( + [string]$Manifest['PackageIdentifier'], + [string]$Manifest['PackageVersion'], + [string]$Manifest['PackageLocale'] + ) -join '|') +} + +function Read-MergedManifests { + param([string] $SnapshotPath) + $databasePath = Join-Path $SnapshotPath 'Public\index.db' + $feedPath = Join-Path $SnapshotPath 'feed' + $versions = Read-VersionDocuments $SnapshotPath + $manifests = [System.Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal) + foreach ($packageId in $versions.Keys) { + foreach ($record in $versions[$packageId].vD) { + if ($record -isnot [System.Collections.IDictionary] -or + $record.rP -isnot [string]) { + throw "Package $packageId contains a version record without a string rP." + } + $manifestPath = Get-ContainedFilePath -Root $feedPath -RelativePath ([string]$record.rP) + $manifestText = [System.Text.UTF8Encoding]::new($false, $true).GetString( + [System.IO.File]::ReadAllBytes($manifestPath) + ) + foreach ($manifest in (ConvertFrom-YamlDocuments $manifestText)) { + if ($manifest -isnot [System.Collections.IDictionary]) { continue } + $identity = Get-ManifestIdentity $manifest + if ($manifests.ContainsKey($identity)) { + throw "Duplicate merged manifest identity: $identity" + } + $manifests.Add($identity, $manifest) + } + } + } + return $manifests +} + +function Read-RawMergedManifestBytes { + param([string] $SnapshotPath) + $feedPath = Join-Path $SnapshotPath 'feed' + $manifestRoot = Join-Path $feedPath 'manifests' + $manifests = [System.Collections.Generic.Dictionary[string, byte[]]]::new([StringComparer]::Ordinal) + if (-not (Test-Path -LiteralPath $manifestRoot -PathType Container)) { + throw "Merged manifest directory is missing: $manifestRoot" + } + foreach ($file in Get-ChildItem -LiteralPath $manifestRoot -Filter '*.yaml' -File -Recurse) { + $relative = [System.IO.Path]::GetRelativePath($feedPath, $file.FullName).Replace('\', '/') + $containedPath = Get-ContainedFilePath -Root $feedPath -RelativePath $relative + $bytes = [System.IO.File]::ReadAllBytes($containedPath) + $text = [System.Text.UTF8Encoding]::new($false, $true).GetString($bytes) + $documents = ConvertFrom-YamlDocuments $text + if ($documents.Count -ne 1 -or $documents[0] -isnot [System.Collections.IDictionary]) { + throw "Merged manifest is not one YAML mapping: $($file.FullName)" + } + $identity = Get-ManifestIdentity $documents[0] + if ($manifests.ContainsKey($identity)) { + throw "Duplicate merged manifest identity: $identity" + } + $manifests.Add($identity, $bytes) + } + return $manifests +} + +$rust = Get-SnapshotMetadata $RustSnapshot +$dotnet = Get-SnapshotMetadata $DotnetSnapshot +if ($rust.Metadata.artifactFormatVersion -ne 2 -or $dotnet.Metadata.artifactFormatVersion -ne 2) { + throw 'Both snapshots must use artifactFormatVersion 2.' +} +if ([string]$rust.Metadata.buildId -cne [string]$dotnet.Metadata.buildId) { + throw 'Rust and .NET build identifiers differ for identical inputs.' +} +Assert-Equal $rust.Metadata.inventory $dotnet.Metadata.inventory 'Canonical raw input inventories differ.' + +$rustIndex = Read-LogicalIndex $rust.Path +$dotnetIndex = Read-LogicalIndex $dotnet.Path +Assert-Equal $rustIndex $dotnetIndex 'WinGet logical database rows or correlation mappings differ.' + +$rustSidecars = Read-CompressedSidecars $rust.Path +$dotnetSidecars = Read-CompressedSidecars $dotnet.Path +Assert-Equal $rustSidecars.Keys $dotnetSidecars.Keys 'Package sets differ in compressed sidecar maps.' +foreach ($packageId in $rustSidecars.Keys) { + if ($rustSidecars[$packageId].Hash -cne $dotnetSidecars[$packageId].Hash -or + -not (Test-ByteArrayEqual $rustSidecars[$packageId].Bytes $dotnetSidecars[$packageId].Bytes)) { + throw "Compressed version sidecar bytes or package hash differ for $packageId." + } +} + +$rustVersionData = Read-VersionDocuments $rust.Path +$dotnetVersionData = Read-VersionDocuments $dotnet.Path +Assert-Equal $rustVersionData $dotnetVersionData 'WinGet version sidecar data differs between implementations.' + +$rustManifests = Read-MergedManifests $rust.Path +$dotnetManifests = Read-MergedManifests $dotnet.Path +Assert-Equal $rustManifests $dotnetManifests 'Canonical merged manifest data differs.' + +$rustManifestBytes = Read-RawMergedManifestBytes $rust.Path +$dotnetManifestBytes = Read-RawMergedManifestBytes $dotnet.Path +Assert-Equal $rustManifestBytes.Keys $dotnetManifestBytes.Keys 'Merged manifest identities differ.' +foreach ($identity in $rustManifestBytes.Keys) { + if (-not (Test-ByteArrayEqual $rustManifestBytes[$identity] $dotnetManifestBytes[$identity])) { + throw "Raw merged manifest bytes differ for $identity." + } +} + +$report = [ordered]@{ + buildId = [string]$rust.Metadata.buildId + packages = $rustIndex.Count + manifests = $rustManifests.Count + logicalIndex = 'equal' + compressedSidecarBytes = 'equal' + versionData = 'equal' + canonicalManifests = 'equal' + rawMergedManifestBytes = 'equal' + rawInputInventory = 'equal' +} +Write-Output ($report | ConvertTo-Json -Depth 20 -Compress) diff --git a/tests/publisher/compare_production_snapshots.py b/tests/publisher/compare_production_snapshots.py deleted file mode 100644 index 3ef2e37..0000000 --- a/tests/publisher/compare_production_snapshots.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python3 -"""Compare two production snapshots by state identity and logical WinGet data.""" - -from __future__ import annotations - -import argparse -import json -import sqlite3 -import sys -from pathlib import Path -from typing import Any - -import yaml - -from build_v2_fixture import decompress_mszip - - -def canonical(value: Any) -> Any: - if isinstance(value, dict): - return {key: canonical(value[key]) for key in sorted(value)} - if isinstance(value, list): - return [canonical(item) for item in value] - return value - - -def snapshot_metadata(snapshot: Path) -> dict[str, Any]: - return json.loads((snapshot / "snapshot.json").read_text(encoding="utf-8")) - - -def read_index(snapshot: Path) -> dict[str, Any]: - database_path = snapshot / "Public" / "index.db" - with sqlite3.connect(f"file:{database_path.as_posix()}?mode=ro", uri=True) as db: - packages = {} - for package_id, name, moniker, version, arp_min, arp_max in db.execute( - "SELECT id,name,moniker,latest_version,arp_min_version,arp_max_version FROM packages" - ): - packages[package_id] = { - "name": name, - "moniker": moniker, - "latestVersion": version, - "arpMinimum": arp_min, - "arpMaximum": arp_max, - "tags": mapped_values(db, package_id, "tags2", "tag", "tags2_map", "tag"), - "commands": mapped_values(db, package_id, "commands2", "command", "commands2_map", "command"), - "pfns": correlation_values(db, package_id, "pfns2", "pfn"), - "productCodes": correlation_values(db, package_id, "productcodes2", "productcode"), - "upgradeCodes": correlation_values(db, package_id, "upgradecodes2", "upgradecode"), - "normalizedNames": correlation_values(db, package_id, "norm_names2", "norm_name"), - "normalizedPublishers": correlation_values(db, package_id, "norm_publishers2", "norm_publisher"), - } - return packages - - -def mapped_values( - db: sqlite3.Connection, - package_id: str, - value_table: str, - value_column: str, - map_table: str, - map_column: str, -) -> list[str]: - query = ( - f"SELECT valuesTable.{value_column} FROM {value_table} AS valuesTable " - f"JOIN {map_table} AS valueMap ON valueMap.{map_column}=valuesTable.rowid " - "JOIN packages AS package ON package.rowid=valueMap.package " - f"WHERE package.id=? ORDER BY valuesTable.{value_column}" - ) - return [row[0] for row in db.execute(query, (package_id,))] - - -def correlation_values( - db: sqlite3.Connection, package_id: str, table: str, column: str -) -> list[str]: - query = ( - f"SELECT value.{column} FROM {table} AS value " - "JOIN packages AS package ON package.rowid=value.package " - f"WHERE package.id=? ORDER BY value.{column}" - ) - return [row[0] for row in db.execute(query, (package_id,))] - - -def read_version_data(snapshot: Path) -> dict[str, Any]: - database_path = snapshot / "Public" / "index.db" - feed = snapshot / "feed" - with sqlite3.connect(f"file:{database_path.as_posix()}?mode=ro", uri=True) as db: - rows = db.execute("SELECT id,hash FROM packages").fetchall() - result: dict[str, Any] = {} - for package_id, package_hash in rows: - sidecar = ( - feed - / "packages" - / package_id - / package_hash.hex()[:8] - / "versionData.mszyml" - ) - document = yaml.safe_load( - decompress_mszip(sidecar.read_bytes()).decode("utf-8") - ) - if not isinstance(document, dict) or not isinstance(document.get("vD"), list): - raise AssertionError(f"invalid version sidecar for package {package_id}") - result[package_id] = canonical(document) - return result - - -def read_compressed_sidecars(snapshot: Path) -> dict[str, tuple[str, bytes]]: - database_path = snapshot / "Public" / "index.db" - feed = snapshot / "feed" - with sqlite3.connect(f"file:{database_path.as_posix()}?mode=ro", uri=True) as db: - rows = db.execute("SELECT id,hash FROM packages").fetchall() - result: dict[str, tuple[str, bytes]] = {} - for package_id, package_hash in rows: - digest = package_hash.hex() - sidecar = ( - feed - / "packages" - / package_id - / digest[:8] - / "versionData.mszyml" - ) - result[package_id] = (digest, sidecar.read_bytes()) - return result - - -def read_merged_manifests(snapshot: Path) -> dict[str, Any]: - feed = snapshot / "feed" - with sqlite3.connect(f"file:{(snapshot / 'Public' / 'index.db').as_posix()}?mode=ro", uri=True) as db: - hashes = [ - (package_id, package_hash.hex()) - for package_id, package_hash in db.execute("SELECT id,hash FROM packages") - ] - result: dict[str, Any] = {} - for package_id, package_hash in hashes: - sidecar = feed / "packages" / package_id / package_hash[:8] / "versionData.mszyml" - version_data = decompress_mszip(sidecar.read_bytes()).decode("utf-8") - records: set[str] = set() - for document in yaml.safe_load_all(version_data): - if not isinstance(document, dict): - continue - records.update( - item["rP"] - for item in document.get("vD", []) - if isinstance(item, dict) and isinstance(item.get("rP"), str) - ) - for relative in records: - manifest_path = feed / Path(relative.replace("/", "\\")) - for manifest in yaml.safe_load_all(manifest_path.read_text(encoding="utf-8")): - if not isinstance(manifest, dict): - continue - key = "|".join( - ( - str(manifest.get("PackageIdentifier", "")), - str(manifest.get("PackageVersion", "")), - str(manifest.get("PackageLocale", "")), - ) - ) - if key in result: - raise AssertionError(f"duplicate merged manifest identity: {key}") - result[key] = canonical(manifest) - return result - - -def read_merged_manifest_bytes(snapshot: Path) -> dict[str, bytes]: - result: dict[str, bytes] = {} - for path in (snapshot / "feed" / "manifests").rglob("*.yaml"): - raw = path.read_bytes() - document = yaml.safe_load(raw) - if not isinstance(document, dict): - raise AssertionError(f"merged manifest is not a mapping: {path}") - key = "|".join( - ( - str(document.get("PackageIdentifier", "")), - str(document.get("PackageVersion", "")), - str(document.get("PackageLocale", "")), - ) - ) - if key in result: - raise AssertionError(f"duplicate merged manifest identity: {key}") - result[key] = raw - return result - - -def compare(left: Path, right: Path) -> dict[str, Any]: - left = left.resolve(strict=True) - right = right.resolve(strict=True) - left_meta = snapshot_metadata(left) - right_meta = snapshot_metadata(right) - if left_meta.get("artifactFormatVersion") != 2 or right_meta.get("artifactFormatVersion") != 2: - raise AssertionError("both snapshots must use artifactFormatVersion 2") - if left_meta.get("buildId") != right_meta.get("buildId"): - raise AssertionError("Rust and .NET build identifiers differ for identical inputs") - if left_meta.get("inventory") != right_meta.get("inventory"): - raise AssertionError("canonical raw input inventories differ") - left_index = read_index(left) - right_index = read_index(right) - if canonical(left_index) != canonical(right_index): - raise AssertionError("WinGet logical database rows or correlation mappings differ") - left_sidecars = read_compressed_sidecars(left) - right_sidecars = read_compressed_sidecars(right) - if left_sidecars != right_sidecars: - differing = sorted( - package_id - for package_id in left_sidecars.keys() | right_sidecars.keys() - if left_sidecars.get(package_id) != right_sidecars.get(package_id) - ) - raise AssertionError( - "compressed version sidecar bytes or package hash differ: " - + ", ".join(differing) - ) - left_version_data = read_version_data(left) - right_version_data = read_version_data(right) - if canonical(left_version_data) != canonical(right_version_data): - raise AssertionError("WinGet version sidecar data differs between implementations") - left_manifests = read_merged_manifests(left) - right_manifests = read_merged_manifests(right) - if canonical(left_manifests) != canonical(right_manifests): - raise AssertionError("canonical merged manifest data differs between implementations") - left_manifest_bytes = read_merged_manifest_bytes(left) - right_manifest_bytes = read_merged_manifest_bytes(right) - if left_manifest_bytes != right_manifest_bytes: - differing = sorted( - key - for key in left_manifest_bytes.keys() | right_manifest_bytes.keys() - if left_manifest_bytes.get(key) != right_manifest_bytes.get(key) - ) - raise AssertionError("raw merged manifest bytes differ: " + ", ".join(differing)) - return { - "buildId": left_meta["buildId"], - "packages": len(left_index), - "manifests": len(left_manifests), - "logicalIndex": "equal", - "compressedSidecarBytes": "equal", - "versionData": "equal", - "canonicalManifests": "equal", - "rawMergedManifestBytes": "equal", - "rawInputInventory": "equal", - } - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--rust-snapshot", type=Path, required=True) - parser.add_argument("--dotnet-snapshot", type=Path, required=True) - args = parser.parse_args() - try: - print( - json.dumps( - compare(args.rust_snapshot, args.dotnet_snapshot), - sort_keys=True, - ) - ) - except Exception as error: - print(f"cross-language snapshot comparison failed: {error}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/publisher/contract.md b/tests/publisher/contract.md index 0cad249..90b8581 100644 --- a/tests/publisher/contract.md +++ b/tests/publisher/contract.md @@ -76,9 +76,10 @@ fixture signer in the Sandbox machine `Root` store; trusting it only in the Sandbox user's `TrustedPeople`/`Root` stores was insufficient (`0x800B0109`). The host certificate stores were not changed. The test adds/updates the source, searches and shows the fixture package, then downloads the harmless payload -and verifies its expected SHA-256 without installing it. A Python SQLite read -and native MSZIP round trip establish additional independent checks, but -neither substitutes for the stock client. +and verifies its expected SHA-256 without installing it. Independent +PowerShell helpers inspect SQLite through the native SQLite API and perform a +Windows Compression API MSZIP round trip. These additional checks do not +substitute for the stock client. ## Non-upstream publisher policy (separate from the independent fixture gate) @@ -140,8 +141,11 @@ schema and does not require this snapshot-format field. Cross-language comparisons pin canonical manifest bytes, raw input inventory, logical SQLite index contents, version-data sidecars and their hashes, and the -snapshot `indexHash`. They also validate the resulting SQLite tables with -Python's `sqlite3` and the sidecar bytes with the Windows native MSZIP API. +index digest recorded by each snapshot against that snapshot's exact +`Public/index.db` bytes (the two SQLite files are not required to be bytewise +identical). The helpers independently inspect SQLite tables through the native +SQLite API and validate sidecar bytes with the Windows native MSZIP API. The +YAML helpers use the existing YamlDotNet assembly from the .NET CLI build. These checks establish the tested outputs and interoperability; the private state schema remains governed by the embedded production schemas. diff --git a/tests/publisher/publisher-validation.ps1 b/tests/publisher/publisher-validation.ps1 new file mode 100644 index 0000000..a9d21c5 --- /dev/null +++ b/tests/publisher/publisher-validation.ps1 @@ -0,0 +1,437 @@ +Set-StrictMode -Version Latest + +function Initialize-PublisherValidation { + param( + [Parameter(Mandatory = $true)][string] $SqliteLibraryPath, + [string] $YamlDotNetPath + ) + + foreach ($path in @($SqliteLibraryPath, $YamlDotNetPath) | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required validation dependency does not exist: $path" + } + } + + if (-not ('PingetPublisher.Validation.NativeSqlite' -as [type])) { + $nativeSqlite = @' +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace PingetPublisher.Validation +{ + public static class NativeSqlite + { + private const int Ok = 0; + private const int Row = 100; + private const int Done = 101; + private const int OpenReadOnly = 0x00000001; + private const int OpenReadWrite = 0x00000002; + private const int OpenCreate = 0x00000004; + private static bool resolverInstalled; + + public static void Initialize(string libraryPath) + { + if (!resolverInstalled) + { + var fullPath = System.IO.Path.GetFullPath(libraryPath); + NativeLibrary.SetDllImportResolver( + typeof(NativeSqlite).Assembly, + (name, assembly, searchPath) => + name == "pinget_validation_sqlite" + ? NativeLibrary.Load(fullPath) + : IntPtr.Zero); + resolverInstalled = true; + } + } + + public static void Execute(string path, string sql) + { + IntPtr database = Open(path, OpenReadWrite | OpenCreate); + IntPtr error = IntPtr.Zero; + try + { + int result = sqlite3_exec(database, sql, IntPtr.Zero, IntPtr.Zero, out error); + if (result != Ok) + { + string message = error == IntPtr.Zero + ? Utf8(sqlite3_errmsg(database)) + : Utf8(error); + throw new InvalidOperationException("SQLite execute failed (" + result + "): " + message); + } + } + finally + { + if (error != IntPtr.Zero) sqlite3_free(error); + sqlite3_close(database); + } + } + + public static List> Query(string path, string sql) + { + IntPtr database = Open(path, OpenReadOnly); + IntPtr statement = IntPtr.Zero; + try + { + int result = sqlite3_prepare_v2(database, sql, -1, out statement, IntPtr.Zero); + Check(database, result, "prepare query"); + var rows = new List>(); + while (true) + { + result = sqlite3_step(statement); + if (result == Done) break; + if (result != Row) Check(database, result, "execute query"); + + int count = sqlite3_column_count(statement); + var row = new Dictionary(StringComparer.Ordinal); + for (int column = 0; column < count; column++) + { + string name = Utf8(sqlite3_column_name(statement, column)); + int type = sqlite3_column_type(statement, column); + object value; + switch (type) + { + case 1: value = sqlite3_column_int64(statement, column); break; + case 2: value = sqlite3_column_double(statement, column); break; + case 3: value = Utf8(sqlite3_column_text(statement, column)); break; + case 4: + int length = sqlite3_column_bytes(statement, column); + var bytes = new byte[length]; + if (length != 0) Marshal.Copy(sqlite3_column_blob(statement, column), bytes, 0, length); + value = bytes; + break; + case 5: value = null; break; + default: throw new InvalidOperationException("Unknown SQLite column type " + type); + } + row.Add(name, value); + } + rows.Add(row); + } + return rows; + } + finally + { + if (statement != IntPtr.Zero) sqlite3_finalize(statement); + sqlite3_close(database); + } + } + + private static IntPtr Open(string path, int flags) + { + int result = sqlite3_open_v2(path, out IntPtr database, flags, IntPtr.Zero); + if (result != Ok) + { + string message = database == IntPtr.Zero ? "no database handle" : Utf8(sqlite3_errmsg(database)); + if (database != IntPtr.Zero) sqlite3_close(database); + throw new InvalidOperationException("SQLite open failed (" + result + "): " + message); + } + return database; + } + + private static void Check(IntPtr database, int result, string operation) + { + if (result != Ok) + throw new InvalidOperationException("SQLite " + operation + " failed (" + result + "): " + Utf8(sqlite3_errmsg(database))); + } + + private static string Utf8(IntPtr value) + { + return value == IntPtr.Zero ? String.Empty : Marshal.PtrToStringUTF8(value); + } + + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_open_v2( + [MarshalAs(UnmanagedType.LPUTF8Str)] string filename, + out IntPtr database, int flags, IntPtr vfs); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_close(IntPtr database); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_exec( + IntPtr database, [MarshalAs(UnmanagedType.LPUTF8Str)] string sql, + IntPtr callback, IntPtr argument, out IntPtr error); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern void sqlite3_free(IntPtr value); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr sqlite3_errmsg(IntPtr database); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_prepare_v2( + IntPtr database, [MarshalAs(UnmanagedType.LPUTF8Str)] string sql, + int byteCount, out IntPtr statement, IntPtr tail); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_step(IntPtr statement); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_finalize(IntPtr statement); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_column_count(IntPtr statement); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr sqlite3_column_name(IntPtr statement, int column); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_column_type(IntPtr statement, int column); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern long sqlite3_column_int64(IntPtr statement, int column); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern double sqlite3_column_double(IntPtr statement, int column); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr sqlite3_column_text(IntPtr statement, int column); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr sqlite3_column_blob(IntPtr statement, int column); + [DllImport("pinget_validation_sqlite", CallingConvention = CallingConvention.Cdecl)] + private static extern int sqlite3_column_bytes(IntPtr statement, int column); + } + + public static class NativeMsZip + { + private const uint AlgorithmMsZip = 2; + private const int ErrorInsufficientBuffer = 122; + + public static byte[] Compress(byte[] input) => Transform(input, true); + public static byte[] Decompress(byte[] input) => Transform(input, false); + + private static byte[] Transform(byte[] input, bool compress) + { + if (input == null) throw new ArgumentNullException(nameof(input)); + IntPtr handle; + bool created = compress + ? CreateCompressor(AlgorithmMsZip, IntPtr.Zero, out handle) + : CreateDecompressor(AlgorithmMsZip, IntPtr.Zero, out handle); + if (!created) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); + try + { + UIntPtr required; + bool sized = compress + ? Compress(handle, input, (UIntPtr)input.Length, null, UIntPtr.Zero, out required) + : Decompress(handle, input, (UIntPtr)input.Length, null, UIntPtr.Zero, out required); + int error = Marshal.GetLastWin32Error(); + if (sized || error != ErrorInsufficientBuffer) + throw new InvalidOperationException("Windows Compression API did not report the required buffer size (error " + error + ")."); + ulong size = required.ToUInt64(); + if (size > Int32.MaxValue || (!compress && size > 64UL * 1024 * 1024)) + throw new InvalidOperationException("Native MSZIP output exceeds the supported validation limit."); + var output = new byte[(int)size]; + UIntPtr written; + bool succeeded = compress + ? Compress(handle, input, (UIntPtr)input.Length, output, (UIntPtr)output.Length, out written) + : Decompress(handle, input, (UIntPtr)input.Length, output, (UIntPtr)output.Length, out written); + if (!succeeded) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); + Array.Resize(ref output, checked((int)written.ToUInt64())); + return output; + } + finally + { + if (compress) CloseCompressor(handle); + else CloseDecompressor(handle); + } + } + + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CreateCompressor(uint algorithm, IntPtr options, out IntPtr compressor); + [DllImport("cabinet.dll", SetLastError = true, EntryPoint = "Compress")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Compress(IntPtr compressor, byte[] input, UIntPtr inputSize, byte[] output, UIntPtr outputSize, out UIntPtr written); + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CloseCompressor(IntPtr compressor); + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CreateDecompressor(uint algorithm, IntPtr options, out IntPtr decompressor); + [DllImport("cabinet.dll", SetLastError = true, EntryPoint = "Decompress")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Decompress(IntPtr decompressor, byte[] input, UIntPtr inputSize, byte[] output, UIntPtr outputSize, out UIntPtr written); + [DllImport("cabinet.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CloseDecompressor(IntPtr decompressor); + } +} +'@ + Add-Type -TypeDefinition $nativeSqlite -Language CSharp -ErrorAction Stop + } + + if (-not ('PingetPublisher.Validation.NativeMsZip' -as [type])) { + throw 'Native MSZIP validation helper did not load.' + } + [PingetPublisher.Validation.NativeSqlite]::Initialize($SqliteLibraryPath) + + if (-not [string]::IsNullOrWhiteSpace($YamlDotNetPath) -and + -not ('YamlDotNet.RepresentationModel.YamlStream' -as [type])) { + Add-Type -Path $YamlDotNetPath -ErrorAction Stop + } +} + +function Get-Sha256Hex { + param([Parameter(Mandatory = $true)][byte[]] $Bytes) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + return ([Convert]::ToHexString($sha.ComputeHash($Bytes))).ToLowerInvariant() + } + finally { + $sha.Dispose() + } +} + +function Get-FileSha256Hex { + param([Parameter(Mandatory = $true)][string] $Path) + return Get-Sha256Hex ([System.IO.File]::ReadAllBytes($Path)) +} + +function Test-ByteArrayEqual { + param([byte[]] $Left, [byte[]] $Right) + if ($null -eq $Left -or $null -eq $Right -or $Left.Length -ne $Right.Length) { + return $false + } + for ($index = 0; $index -lt $Left.Length; $index++) { + if ($Left[$index] -ne $Right[$index]) { return $false } + } + return $true +} + +function ConvertFrom-YamlNode { + param([Parameter(Mandatory = $true)][YamlDotNet.RepresentationModel.YamlNode] $Node) + + if ($Node -is [YamlDotNet.RepresentationModel.YamlScalarNode]) { + $scalar = [YamlDotNet.RepresentationModel.YamlScalarNode]$Node + if ($scalar.Tag -eq 'tag:yaml.org,2002:null' -or + ($scalar.Style -eq [YamlDotNet.Core.ScalarStyle]::Plain -and + ($scalar.Value -eq '' -or $scalar.Value -eq '~' -or $scalar.Value -eq 'null' -or $scalar.Value -eq 'Null' -or $scalar.Value -eq 'NULL'))) { + return $null + } + return [string]$scalar.Value + } + if ($Node -is [YamlDotNet.RepresentationModel.YamlSequenceNode]) { + $items = [System.Collections.Generic.List[object]]::new() + foreach ($child in ([YamlDotNet.RepresentationModel.YamlSequenceNode]$Node).Children) { + $items.Add((ConvertFrom-YamlNode $child)) + } + return ,$items.ToArray() + } + if ($Node -is [YamlDotNet.RepresentationModel.YamlMappingNode]) { + $map = [System.Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal) + foreach ($entry in ([YamlDotNet.RepresentationModel.YamlMappingNode]$Node).Children.GetEnumerator()) { + if ($entry.Key -isnot [YamlDotNet.RepresentationModel.YamlScalarNode]) { + throw 'Independent YAML validation does not permit non-scalar mapping keys.' + } + $key = [string]([YamlDotNet.RepresentationModel.YamlScalarNode]$entry.Key).Value + if ($map.ContainsKey($key)) { throw "Duplicate YAML mapping key: $key" } + $map.Add($key, (ConvertFrom-YamlNode $entry.Value)) + } + return $map + } + throw "Unsupported YAML node type: $($Node.GetType().FullName)" +} + +function ConvertFrom-YamlDocuments { + param([Parameter(Mandatory = $true)][string] $Text) + $stream = [YamlDotNet.RepresentationModel.YamlStream]::new() + $reader = [System.IO.StringReader]::new($Text) + try { + $stream.Load($reader) + $documents = [System.Collections.Generic.List[object]]::new() + foreach ($document in $stream.Documents) { + $documents.Add((ConvertFrom-YamlNode $document.RootNode)) + } + return ,$documents.ToArray() + } + finally { + $reader.Dispose() + } +} + +function ConvertTo-CanonicalValue { + param([AllowNull()][object] $Value) + if ($null -eq $Value) { return $null } + if ($Value -is [System.Collections.IDictionary]) { + $map = [System.Collections.Generic.SortedDictionary[string, object]]::new([StringComparer]::Ordinal) + foreach ($key in $Value.Keys) { + $map.Add([string]$key, (ConvertTo-CanonicalValue $Value[$key])) + } + return $map + } + if ($Value -is [System.Collections.IEnumerable] -and $Value -isnot [string] -and $Value -isnot [byte[]]) { + $items = [System.Collections.Generic.List[object]]::new() + foreach ($item in $Value) { $items.Add((ConvertTo-CanonicalValue $item)) } + return ,$items.ToArray() + } + return $Value +} + +function ConvertTo-CanonicalJson { + param([AllowNull()][object] $Value) + return ConvertTo-Json -InputObject (ConvertTo-CanonicalValue $Value) -Depth 100 -Compress +} + +function Get-ContainedFilePath { + param( + [Parameter(Mandatory = $true)][string] $Root, + [Parameter(Mandatory = $true)][string] $RelativePath + ) + if ([System.IO.Path]::IsPathRooted($RelativePath) -or + $RelativePath -match '(^|[\\/])\.\.([\\/]|$)') { + throw "Artifact path is not a safe relative path: $RelativePath" + } + if (-not (Test-Path -LiteralPath $Root -PathType Container)) { + throw "Artifact root is not an existing directory: $Root" + } + $rootPath = [System.IO.Path]::GetFullPath($Root).TrimEnd([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) + $volumeRoot = [System.IO.Path]::GetPathRoot($rootPath) + $rootTail = $rootPath.Substring($volumeRoot.Length) + $rootCursor = $volumeRoot + foreach ($rootPart in $rootTail.Split([System.IO.Path]::DirectorySeparatorChar, [System.StringSplitOptions]::RemoveEmptyEntries)) { + $rootCursor = Join-Path $rootCursor $rootPart + $rootAttributes = [System.IO.File]::GetAttributes($rootCursor) + if (($rootAttributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Artifact root traverses a reparse point: $rootCursor" + } + } + $candidate = [System.IO.Path]::GetFullPath((Join-Path $rootPath $RelativePath.Replace('/', [System.IO.Path]::DirectorySeparatorChar))) + $prefix = $rootPath + [System.IO.Path]::DirectorySeparatorChar + if (-not $candidate.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Artifact path escapes its snapshot root: $RelativePath" + } + $cursor = $rootPath + $relativeTail = $candidate.Substring($prefix.Length) + $parts = $relativeTail.Split([System.IO.Path]::DirectorySeparatorChar, [System.StringSplitOptions]::RemoveEmptyEntries) + foreach ($part in $parts) { + $cursor = Join-Path $cursor $part + if (-not (Test-Path -LiteralPath $cursor)) { throw "Artifact file does not exist: $RelativePath" } + $attributes = [System.IO.File]::GetAttributes($cursor) + if (($attributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Artifact path traverses a reparse point: $RelativePath" + } + } + if (([System.IO.File]::GetAttributes($candidate) -band [System.IO.FileAttributes]::Directory) -ne 0) { + throw "Artifact path is not a file: $RelativePath" + } + return $candidate +} + +function Get-PackageSidecar { + param([string] $FeedPath, [string] $PackageId, [byte[]] $HashBytes) + if ($HashBytes.Length -ne 32) { throw "Package $PackageId sidecar hash is not 32 bytes." } + $hash = ([Convert]::ToHexString($HashBytes)).ToLowerInvariant() + $relative = "packages/$PackageId/$($hash.Substring(0, 8))/versionData.mszyml" + return [pscustomobject]@{ + Hash = $hash + RelativePath = $relative + Path = Get-ContainedFilePath -Root $FeedPath -RelativePath $relative + } +} + +function Get-SqliteTableNames { + param([string] $DatabasePath) + $rows = [PingetPublisher.Validation.NativeSqlite]::Query( + $DatabasePath, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + return @($rows | ForEach-Object { [string]$_.name }) +} + +function Assert-Equal { + param([object] $Actual, [object] $Expected, [string] $Message) + $actualJson = ConvertTo-CanonicalJson $Actual + $expectedJson = ConvertTo-CanonicalJson $Expected + if ($actualJson -cne $expectedJson) { + throw "$Message`nExpected: $expectedJson`nActual: $actualJson" + } +} diff --git a/tests/publisher/run-production-conformance.ps1 b/tests/publisher/run-production-conformance.ps1 index a052c9f..5b17abc 100644 --- a/tests/publisher/run-production-conformance.ps1 +++ b/tests/publisher/run-production-conformance.ps1 @@ -6,7 +6,8 @@ param( [string] $InputDirectory = (Join-Path $PSScriptRoot 'fixtures\multi'), [string] $WorkDirectory = (Join-Path $env:TEMP ('PingetPublisherConformance-' + [Guid]::NewGuid().ToString('N'))), [string] $ExpectedPackageId = 'Pinget.Test.Matrix', - [string] $Python = 'python', + [string] $SqliteLibraryPath = '', + [string] $YamlDotNetPath = '', [string] $MakeAppxPath = 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64\makeappx.exe', [string] $PackageName = 'Pinget.Publisher.Conformance', [string] $Publisher = 'CN=Pinget Publisher Conformance Test' @@ -23,8 +24,52 @@ $dotnetExecutablePath = if ([string]::IsNullOrWhiteSpace($DotnetExecutable)) { } else { [System.IO.Path]::GetFullPath($DotnetExecutable) } -$validator = Join-Path $PSScriptRoot 'validate_production_snapshot.py' -$comparer = Join-Path $PSScriptRoot 'compare_production_snapshots.py' +$validator = Join-Path $PSScriptRoot 'validate_production_snapshot.ps1' +$comparer = Join-Path $PSScriptRoot 'compare_production_snapshots.ps1' + +function Resolve-PublisherDependency { + param([string] $ExplicitPath, [string] $FileName, [string[]] $SearchRoots) + if (-not [string]::IsNullOrWhiteSpace($ExplicitPath)) { + $resolved = [System.IO.Path]::GetFullPath($ExplicitPath) + if (-not (Test-Path -LiteralPath $resolved -PathType Leaf)) { + throw "Required validation dependency does not exist: $resolved" + } + return $resolved + } + foreach ($root in $SearchRoots | Where-Object { -not [string]::IsNullOrWhiteSpace($_) -and (Test-Path -LiteralPath $_ -PathType Container) }) { + $candidate = Join-Path $root $FileName + if (Test-Path -LiteralPath $candidate -PathType Leaf) { return $candidate } + $nestedFiles = @(Get-ChildItem -LiteralPath $root -Filter $FileName -File -Recurse -ErrorAction SilentlyContinue) + if ($FileName -eq 'e_sqlite3.dll') { + $runtime = switch ([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture) { + 'X64' { 'win-x64' } + 'Arm64' { 'win-arm64' } + 'X86' { 'win-x86' } + default { throw "Unsupported architecture for native SQLite inspection: $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture)" } + } + $preferred = $nestedFiles | Where-Object { + $_.FullName -match ([regex]::Escape("\runtimes\$runtime\native\e_sqlite3.dll") + '$') + } | Select-Object -First 1 + if ($null -ne $preferred) { return $preferred.FullName } + } + $nested = $nestedFiles | Select-Object -First 1 + if ($null -ne $nested) { return $nested.FullName } + } + throw "Could not locate $FileName in the existing .NET CLI build outputs. Pass its path explicitly." +} + +$dotnetOutputDirectory = if (-not [string]::IsNullOrWhiteSpace($dotnetExecutablePath)) { + Split-Path -Parent $dotnetExecutablePath +} else { + Split-Path -Parent $dotnetPath +} +$managedCliDependencies = Join-Path $repositoryRoot 'dotnet\src\Devolutions.Pinget.Cli\bin\Release\net10.0' +$validationRoots = @($dotnetOutputDirectory, $managedCliDependencies) | + Select-Object -Unique +$sqlitePath = Resolve-PublisherDependency $SqliteLibraryPath 'e_sqlite3.dll' $validationRoots +$yamlPath = Resolve-PublisherDependency $YamlDotNetPath 'YamlDotNet.dll' $validationRoots +. (Join-Path $PSScriptRoot 'publisher-validation.ps1') +Initialize-PublisherValidation -SqliteLibraryPath $sqlitePath -YamlDotNetPath $yamlPath foreach ($required in @($inputPath, $rustPath, $validator, $comparer, $MakeAppxPath)) { if (-not (Test-Path -LiteralPath $required)) { @@ -168,8 +213,20 @@ foreach ($implementation in @('Rust', 'Dotnet')) { } } 'yaml' { - if ($formatted.Text -notmatch "(?m)^lastBuildId:\s*['""]?$([regex]::Escape($rustBuildId))['""]?\s*$") { - throw "$implementation YAML status omitted the current build id: $($formatted.Text)" + $yamlDocuments = ConvertFrom-YamlDocuments $formatted.Text + if ($yamlDocuments.Count -ne 1 -or + $yamlDocuments[0] -isnot [System.Collections.IDictionary]) { + throw "$implementation YAML status is not one mapping document: $($formatted.Text)" + } + $yamlStatus = $yamlDocuments[0] + $expectedYamlFields = @('buildCount', 'lastBuildId', 'lastBuildTimestamp', 'lastPublishedBuildId') + if ((@($yamlStatus.Keys | Sort-Object) -join "`n") -cne + (@($expectedYamlFields | Sort-Object) -join "`n") -or + $yamlStatus.lastBuildId -cne $rustBuildId -or + $null -ne $yamlStatus.lastPublishedBuildId -or + $yamlStatus.buildCount -cne '1' -or + [string]::IsNullOrWhiteSpace([string]$yamlStatus.lastBuildTimestamp)) { + throw "$implementation YAML status has unexpected fields or values: $($formatted.Text)" } } 'text' { @@ -218,15 +275,11 @@ foreach ($doctorReport in @($rustDoctor, $dotnetDoctor)) { $rustSnapshot = Join-Path (Join-Path $rustState 'builds') $rustBuildId $dotnetSnapshot = Join-Path (Join-Path $dotnetState 'builds') $dotnetBuildId foreach ($snapshot in @($rustSnapshot, $dotnetSnapshot)) { - & $Python -B $validator --snapshot $snapshot --expect-package $ExpectedPackageId - if ($LASTEXITCODE -ne 0) { - throw "Independent SQLite/MSZIP snapshot validation failed for $snapshot (exit $LASTEXITCODE)." - } -} -& $Python -B $comparer --rust-snapshot $rustSnapshot --dotnet-snapshot $dotnetSnapshot -if ($LASTEXITCODE -ne 0) { - throw "Cross-language logical snapshot comparison failed (exit $LASTEXITCODE)." + & $validator -Snapshot $snapshot -ExpectPackage $ExpectedPackageId ` + -SqliteLibraryPath $sqlitePath -YamlDotNetPath $yamlPath } +& $comparer -RustSnapshot $rustSnapshot -DotnetSnapshot $dotnetSnapshot ` + -SqliteLibraryPath $sqlitePath -YamlDotNetPath $yamlPath $rustDiff = ConvertFrom-PingetJson ` (Invoke-Pinget 'Rust' ($common + @('diff', '--input', $inputPath, '--state', $rustState))) 'Rust diff' @@ -278,7 +331,7 @@ Assert-VerifiedReport $rustVerifyDotnetDestination $dotnetBuildId 'Rust destinat WorkDirectory = $workPath RequiredArtifacts = @( 'Rust and .NET status/verify can reopen each other''s state' - 'Python sqlite3 and native Windows MSZIP validation passed for both snapshots' + 'Independent native SQLite and Windows MSZIP validation passed for both snapshots' 'Canonicalized package, correlation, manifest and input-inventory data matched' 'Rust and .NET reciprocal destination verification passed for unsigned MSIX outputs' ) diff --git a/tests/publisher/validate_production_snapshot.ps1 b/tests/publisher/validate_production_snapshot.ps1 new file mode 100644 index 0000000..4f906a6 --- /dev/null +++ b/tests/publisher/validate_production_snapshot.ps1 @@ -0,0 +1,161 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string] $Snapshot, + [string] $ExpectPackage, + [Parameter(Mandatory = $true)][string] $SqliteLibraryPath, + [Parameter(Mandatory = $true)][string] $YamlDotNetPath +) + +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'publisher-validation.ps1') +Initialize-PublisherValidation -SqliteLibraryPath $SqliteLibraryPath -YamlDotNetPath $YamlDotNetPath + +$snapshotPath = [System.IO.Path]::GetFullPath((Resolve-Path -LiteralPath $Snapshot).Path) +$metadataPath = Join-Path $snapshotPath 'snapshot.json' +if (-not (Test-Path -LiteralPath $metadataPath -PathType Leaf)) { + throw "Snapshot metadata is missing: $metadataPath" +} +$metadata = Get-Content -LiteralPath $metadataPath -Raw | ConvertFrom-Json -AsHashtable +if ($metadata.schemaVersion -ne 1) { throw 'Snapshot schemaVersion is not 1.' } +if ($metadata.artifactFormatVersion -ne 2) { throw 'Snapshot artifactFormatVersion is not 2.' } +if ([string]::IsNullOrWhiteSpace([string]$metadata.buildId)) { throw 'Snapshot buildId is missing.' } +if ($metadata.artifacts -isnot [System.Collections.IDictionary]) { + throw 'Snapshot artifacts must be a JSON object.' +} + +$feedPath = Join-Path $snapshotPath 'feed' +foreach ($relative in $metadata.artifacts.Keys) { + $artifactPath = Get-ContainedFilePath -Root $feedPath -RelativePath ([string]$relative) + $actualHash = Get-FileSha256Hex $artifactPath + if ($actualHash -cne [string]$metadata.artifacts[$relative]) { + throw "Artifact hash mismatch: $relative" + } +} + +$indexPath = Join-Path $snapshotPath 'Public\index.db' +if (-not (Test-Path -LiteralPath $indexPath -PathType Leaf)) { + throw "Snapshot WinGet index is missing: $indexPath" +} +$indexHash = [string]$metadata.indexHash +if ($indexHash -cnotmatch '^[0-9a-f]{64}$' -or + (Get-FileSha256Hex $indexPath) -cne $indexHash) { + throw 'Snapshot indexHash does not match the exact Public/index.db bytes.' +} +$integrity = [PingetPublisher.Validation.NativeSqlite]::Query($indexPath, 'PRAGMA integrity_check') +if ($integrity.Count -ne 1 -or $integrity[0].integrity_check -ne 'ok') { + throw "SQLite integrity_check failed: $($integrity | ConvertTo-Json -Compress)" +} +$expectedTables = @('commands2', 'commands2_map', 'metadata', 'norm_names2', 'norm_publishers2', 'packages', 'pfns2', 'productcodes2', 'tags2', 'tags2_map', 'upgradecodes2') +Assert-Equal (Get-SqliteTableNames $indexPath) $expectedTables 'Unexpected WinGet v2 table set.' +$indexMetadata = [PingetPublisher.Validation.NativeSqlite]::Query($indexPath, 'SELECT name, value FROM metadata') +$indexMetadataMap = @{} +foreach ($row in $indexMetadata) { $indexMetadataMap[[string]$row.name] = [string]$row.value } +if ($indexMetadataMap.majorVersion -ne '2' -or $indexMetadataMap.minorVersion -ne '0') { + throw 'WinGet index schema markers are not majorVersion=2 and minorVersion=0.' +} + +$packages = [PingetPublisher.Validation.NativeSqlite]::Query( + $indexPath, + 'SELECT id, name, latest_version, hash, arp_min_version, arp_max_version FROM packages ORDER BY id COLLATE NOCASE' +) +if ($packages.Count -eq 0) { throw 'Production snapshot contains no packages.' } +if (-not [string]::IsNullOrEmpty($ExpectPackage) -and + @($packages | Where-Object { $_.id -ceq $ExpectPackage }).Count -eq 0) { + throw "Expected package '$ExpectPackage' is absent." +} + +$decodedVersions = [ordered]@{} +foreach ($package in $packages) { + $packageId = [string]$package.id + if ($package.hash -isnot [byte[]] -or $package.hash.Length -ne 32) { + throw "Package $packageId has no 32-byte sidecar hash." + } + $sidecar = Get-PackageSidecar -FeedPath $feedPath -PackageId $packageId -HashBytes $package.hash + if ((Get-FileSha256Hex $sidecar.Path) -cne $sidecar.Hash) { + throw "Package $packageId sidecar hash/path is invalid." + } + $rawVersionData = [PingetPublisher.Validation.NativeMsZip]::Decompress( + [System.IO.File]::ReadAllBytes($sidecar.Path) + ) + $versionText = [System.Text.UTF8Encoding]::new($false, $true).GetString($rawVersionData) + if (-not $versionText.StartsWith("sV: `"1.0`"`n", [StringComparison]::Ordinal) -and + -not $versionText.StartsWith("sV: `"1.0`"`r`n", [StringComparison]::Ordinal)) { + throw "Package $packageId has an unexpected version sidecar header." + } + $versionDocuments = ConvertFrom-YamlDocuments $versionText + if ($versionDocuments.Count -ne 1 -or $versionDocuments[0] -isnot [System.Collections.IDictionary]) { + throw "Package $packageId version sidecar is not one mapping document." + } + $versionDocument = $versionDocuments[0] + if ($versionDocument.sV -cne '1.0' -or $versionDocument.vD -isnot [System.Array] -or + $versionDocument.vD.Length -eq 0) { + throw "Package $packageId sidecar has an invalid schema marker or empty vD list." + } + + $versions = [System.Collections.Generic.List[string]]::new() + $arpMinimums = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $arpMaximums = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($record in $versionDocument.vD) { + if ($record -isnot [System.Collections.IDictionary]) { + throw "Package $packageId sidecar contains a non-object vD record." + } + foreach ($field in @('v', 'rP', 's256H')) { + if ($record[$field] -isnot [string]) { + throw "Package $packageId sidecar record lacks string $field." + } + } + $version = [string]$record.v + $manifestRelative = [string]$record.rP + $manifestHash = [string]$record.s256H + if ($manifestHash -cnotmatch '^[0-9a-f]{64}$') { + throw "Package $packageId sidecar contains invalid lowercase SHA-256 for $version." + } + if (-not $versions.Contains($version)) { + $versions.Add($version) + } else { + throw "Package $packageId sidecar contains duplicate version $version." + } + $manifestPath = Get-ContainedFilePath -Root $feedPath -RelativePath $manifestRelative + if ((Get-FileSha256Hex $manifestPath) -cne $manifestHash) { + throw "Package $packageId manifest hash mismatch for $version." + } + if ($record.ContainsKey('aMiV') -and $record.aMiV -is [string]) { [void]$arpMinimums.Add([string]$record.aMiV) } + if ($record.ContainsKey('aMaV') -and $record.aMaV -is [string]) { [void]$arpMaximums.Add([string]$record.aMaV) } + } + if ($versions[0] -cne [string]$package.latest_version) { + throw "Package $packageId sidecar does not place latest version first." + } + if ($null -ne $package.arp_min_version -and -not $arpMinimums.Contains([string]$package.arp_min_version)) { + throw "Minimum ARP version is missing from sidecar for $packageId." + } + if ($null -ne $package.arp_max_version -and -not $arpMaximums.Contains([string]$package.arp_max_version)) { + throw "Maximum ARP version is missing from sidecar for $packageId." + } + if ($packageId -ceq 'Pinget.Test.Matrix') { + $actualArp = @{} + foreach ($record in $versionDocument.vD) { + $actualArp[[string]$record.v] = @($record.aMiV, $record.aMaV) + } + $expectedArp = @{ + '1.9.0' = @('1.9.0', '1.9.0') + '1.10.0' = @('1.10.0', '1.10.0') + } + Assert-Equal $actualArp $expectedArp 'Per-version ARP metadata is incorrect for Pinget.Test.Matrix.' + if ($package.arp_min_version -cne '1.9.0' -or $package.arp_max_version -cne '1.10.0') { + throw 'Package-level ARP bounds are incorrect for Pinget.Test.Matrix.' + } + } + $decodedVersions[$packageId] = @($versions.ToArray()) +} + +$report = [ordered]@{ + snapshot = $snapshotPath + buildId = [string]$metadata.buildId + packages = [ordered]@{} + versions = $decodedVersions + artifacts = $metadata.artifacts.Count + sqlite = 'ok (native independent SQLite API)' + mszip = 'native Windows Compression API' +} +foreach ($package in $packages) { $report.packages[[string]$package.id] = [string]$package.latest_version } +Write-Output ($report | ConvertTo-Json -Depth 20 -Compress) diff --git a/tests/publisher/validate_production_snapshot.py b/tests/publisher/validate_production_snapshot.py deleted file mode 100644 index ce60dcf..0000000 --- a/tests/publisher/validate_production_snapshot.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 -"""Independently validate a Pinget build snapshot with sqlite3 and native MSZIP.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -import sqlite3 -import sys -from pathlib import Path - -import yaml - -from build_v2_fixture import decompress_mszip - - -EXPECTED_TABLES = { - "metadata", - "packages", - "tags2", - "tags2_map", - "commands2", - "commands2_map", - "pfns2", - "productcodes2", - "norm_names2", - "norm_publishers2", - "upgradecodes2", -} - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def contained_file(root: Path, relative: str) -> Path: - root = root.resolve(strict=True) - path = (root / Path(relative.replace("/", "\\"))).resolve(strict=True) - if not path.is_relative_to(root) or not path.is_file(): - raise AssertionError(f"artifact path is not a contained file: {relative}") - return path - - -def version_records(text: str) -> list[dict[str, object]]: - document = yaml.safe_load(text) - if not isinstance(document, dict) or document.get("sV") != "1.0": - raise AssertionError("version sidecar has an invalid document or schema marker") - records = document.get("vD") - if not isinstance(records, list) or not records: - raise AssertionError("version sidecar has no vD records") - for record in records: - if not isinstance(record, dict): - raise AssertionError("version sidecar contains a non-object vD record") - if not all(isinstance(record.get(key), str) for key in ("v", "rP", "s256H")): - raise AssertionError("version sidecar record lacks string v, rP, or s256H fields") - if not re.fullmatch(r"[0-9a-f]{64}", record["s256H"]): - raise AssertionError("version sidecar contains an invalid lowercase SHA-256") - return records - - -def validate(snapshot: Path, expected_package: str | None) -> dict[str, object]: - snapshot = snapshot.resolve(strict=True) - metadata_path = snapshot / "snapshot.json" - metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - if metadata.get("schemaVersion") != 1: - raise AssertionError("snapshot schemaVersion is not 1") - if metadata.get("artifactFormatVersion") != 2: - raise AssertionError("snapshot artifactFormatVersion is not 2") - if not metadata.get("buildId"): - raise AssertionError("snapshot buildId is missing") - - feed = snapshot / "feed" - artifact_inventory = metadata.get("artifacts") - if not isinstance(artifact_inventory, dict): - raise AssertionError("snapshot artifacts must be a JSON object") - for relative, expected_hash in artifact_inventory.items(): - path = contained_file(feed, relative) - if digest(path) != expected_hash: - raise AssertionError(f"artifact hash mismatch: {relative}") - - index = snapshot / "Public" / "index.db" - with sqlite3.connect(f"file:{index.as_posix()}?mode=ro", uri=True) as db: - integrity = db.execute("PRAGMA integrity_check").fetchone()[0] - if integrity != "ok": - raise AssertionError(f"SQLite integrity_check returned {integrity!r}") - tables = { - row[0] - for row in db.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" - ) - } - if tables != EXPECTED_TABLES: - raise AssertionError(f"unexpected WinGet v2 table set: {sorted(tables)}") - metadata_rows = dict(db.execute("SELECT name,value FROM metadata")) - if (metadata_rows.get("majorVersion"), metadata_rows.get("minorVersion")) != ("2", "0"): - raise AssertionError("WinGet schema markers are not 2.0") - packages = db.execute( - "SELECT id,name,latest_version,hash,arp_min_version,arp_max_version " - "FROM packages ORDER BY id COLLATE NOCASE" - ).fetchall() - if not packages: - raise AssertionError("the production snapshot contains no packages") - if expected_package is not None and expected_package not in {row[0] for row in packages}: - raise AssertionError(f"expected package {expected_package!r} is absent") - - decoded_versions: dict[str, list[str]] = {} - for package_id, _name, latest_version, package_hash, _, _ in packages: - if package_hash is None or len(package_hash) != 32: - raise AssertionError(f"package {package_id} has no 32-byte sidecar hash") - full_hash = package_hash.hex() - sidecar_relative = ( - f"packages/{package_id}/{full_hash[:8]}/versionData.mszyml" - ) - sidecar_path = contained_file(feed, sidecar_relative) - if digest(sidecar_path) != full_hash: - raise AssertionError(f"package {package_id} sidecar hash/path is invalid") - raw_version_data = decompress_mszip(sidecar_path.read_bytes()) - version_text = raw_version_data.decode("utf-8") - if not version_text.startswith('sV: "1.0"\n'): - raise AssertionError(f"package {package_id} has an unexpected sidecar header") - records = version_records(version_text) - record_versions = [record["v"] for record in records] - if len(record_versions) != len(set(record_versions)): - raise AssertionError(f"package {package_id} has duplicate version records") - if record_versions[0] != latest_version: - raise AssertionError( - f"package {package_id} sidecar does not place latest version first" - ) - for record in records: - version = record["v"] - manifest_relative = record["rP"] - manifest_hash = record["s256H"] - manifest_path = contained_file(feed, manifest_relative) - if digest(manifest_path) != manifest_hash: - raise AssertionError( - f"package {package_id} manifest hash mismatch for {version}" - ) - if latest_version not in {record["v"] for record in records}: - raise AssertionError(f"latest version missing from sidecar for {package_id}") - arp_minimum, arp_maximum = next( - row[4:6] for row in packages if row[0] == package_id - ) - sidecar_minima = { - record.get("aMiV") - for record in records - if isinstance(record.get("aMiV"), str) - } - sidecar_maxima = { - record.get("aMaV") - for record in records - if isinstance(record.get("aMaV"), str) - } - if arp_minimum is not None and arp_minimum not in sidecar_minima: - raise AssertionError(f"minimum ARP version missing from sidecar for {package_id}") - if arp_maximum is not None and arp_maximum not in sidecar_maxima: - raise AssertionError(f"maximum ARP version missing from sidecar for {package_id}") - if package_id == "Pinget.Test.Matrix": - expected_arp = { - "1.9.0": ("1.9.0", "1.9.0"), - "1.10.0": ("1.10.0", "1.10.0"), - } - actual_arp = { - record["v"]: (record.get("aMiV"), record.get("aMaV")) - for record in records - } - if actual_arp != expected_arp or (arp_minimum, arp_maximum) != ( - "1.9.0", - "1.10.0", - ): - raise AssertionError( - f"per-version ARP metadata or package bounds are incorrect for {package_id}" - ) - decoded_versions[package_id] = record_versions - - return { - "snapshot": str(snapshot), - "buildId": metadata["buildId"], - "packages": {row[0]: row[2] for row in packages}, - "versions": decoded_versions, - "artifacts": len(artifact_inventory), - "sqlite": "ok", - "mszip": "native Windows Compression API", - } - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--snapshot", type=Path, required=True) - parser.add_argument("--expect-package") - args = parser.parse_args() - try: - print(json.dumps(validate(args.snapshot, args.expect_package), sort_keys=True)) - except Exception as error: - print(f"snapshot validation failed: {error}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())