From 049b6927339c697d4ce1faa4337694cc01e88d29 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 26 Aug 2026 14:46:03 -0400 Subject: [PATCH] LT-22728: Scope local libraries to one build A locally packed library used to reuse the published version string, and NuGet resolves an already-extracted (id, version) before it consults a folder feed. A local pack could therefore be shadowed by the published package, or shadow it, with nothing to tell them apart. SIL.Machine was the clearest case: it has no GitVersion, so it packed as a flat 3.9.2, identical to the package on nuget.org. Derive each pack's version from the checkout instead, as -., taking the core from the library's own GitVersion where it has one. GitVersion.MsBuild assigns Version inside a target, which outranks a command-line property, so it is switched off for the pack. Because a clean commit identifies its contents, a second build from the same commit reuses the package already in the feed. An uncommitted checkout has no stable identity, so it is marked dirty, repacked every time, and the build names the paths responsible. Write the selected versions and the feed to a generated Build/LocalLibraries.props that Build/SilVersions.props imports, rather than passing them on the command line. The restore in Build/PackageRestore.targets runs through Exec, which starts an MSBuild process that does not inherit global properties, so a version passed that way never reached it: the build reported using a local library while every project resolved the published one. Build each library before packing it. A package may include output from a target framework its own project does not build, and pack alone does not produce those, which left L10NSharp unpackable. Keep the feed inside the working tree as .localfeed. A machine-wide feed let one working tree's build delete packages another had just produced, which is also why the existing cleanup could not be trusted; scoped to one working tree, it can be. Skip that cleanup when the build will not restore, so it cannot remove packages nothing will put back. Add Setup-LocalLibraries.ps1 to make a library branch available. It finds the checkout beside FieldWorks or through the library's path variable, and uses an existing worktree for the branch where there is one, since git refuses to check a branch out twice and that worktree may hold work in progress. It fetches but never merges, never switches a branch in a checkout that already has one, and never prompts. Leave inherited package sources in place: a local build adds its own feed for that build only, and the versions it packs cannot collide. Read the cache metadata defensively. Version 1 records no source, which under Set-StrictMode ended the build, and a version directory with no metadata is a partial extraction rather than something to keep. Report every failing assertion instead of stopping at the first, and cover Build with the PowerShell compatibility check, which scanned only Build/Agent. Verified through build.ps1 against liblcm: 114 projects resolved the local package where none did before, and an ordinary build then restored the published one without redownloading it. --- .gitignore | 5 + Build/Agent/powershell-compat.ps1 | 11 +- Build/LocalLibraries.Tests.ps1 | 343 ++++++++++++ Build/LocalLibraries.psm1 | 547 +++++++++++++++++++ Build/Manage-LocalLibraries.ps1 | 330 ++++++----- Build/Setup-LocalLibraries.ps1 | 91 +++ Build/SilVersions.props | 9 + Docs/architecture/dependencies.md | 10 +- Docs/architecture/local-library-debugging.md | 191 +++---- build.ps1 | 99 +++- nuget.config | 6 +- test.ps1 | 9 + 12 files changed, 1352 insertions(+), 299 deletions(-) create mode 100644 Build/LocalLibraries.Tests.ps1 create mode 100644 Build/LocalLibraries.psm1 create mode 100644 Build/Setup-LocalLibraries.ps1 diff --git a/.gitignore b/.gitignore index a4c7c89ff4..1ebb84e0b8 100644 --- a/.gitignore +++ b/.gitignore @@ -223,3 +223,8 @@ DataTreeTimingBaselines.json Docs/migration/working/ Build/Agent/comment-hygiene-report.json .review/ + +# Working-tree-local NuGet feed for -LocalLibraries packs +.localfeed/ +# Generated by build.ps1 -LocalLibraries; selects local library versions +Build/LocalLibraries.props diff --git a/Build/Agent/powershell-compat.ps1 b/Build/Agent/powershell-compat.ps1 index dac023085e..159aa20cfc 100644 --- a/Build/Agent/powershell-compat.ps1 +++ b/Build/Agent/powershell-compat.ps1 @@ -48,11 +48,14 @@ $ErrorActionPreference = 'Stop' Import-Module (Join-Path $PSScriptRoot 'CommentHygiene.psm1') -Force $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path -# Every PowerShell file under Build/Agent, so a new script is covered the day it -# lands rather than when someone remembers to name it here. -$targetFiles = @(Get-ChildItem -LiteralPath $PSScriptRoot -Recurse -File | +# Every PowerShell file under Build, not just Build/Agent: build.ps1 loads +# modules from there under Windows PowerShell 5.1 in CI. +$scanRoots = @($PSScriptRoot, (Join-Path $repoRoot 'Build')) +$targetFiles = @($scanRoots | ForEach-Object { + Get-ChildItem -LiteralPath $_ -Recurse -File -ErrorAction SilentlyContinue + } | Where-Object { $_.Extension -eq '.ps1' -or $_.Extension -eq '.psm1' } | - ForEach-Object { $_.FullName } | Sort-Object) + ForEach-Object { $_.FullName } | Sort-Object -Unique) $violations = New-Object System.Collections.ArrayList diff --git a/Build/LocalLibraries.Tests.ps1 b/Build/LocalLibraries.Tests.ps1 new file mode 100644 index 0000000000..72e2b6bed5 --- /dev/null +++ b/Build/LocalLibraries.Tests.ps1 @@ -0,0 +1,343 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$failures = New-Object System.Collections.ArrayList +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ( + 'FieldWorksLocalLibrariesTests_' + [System.Guid]::NewGuid().ToString('N')) + +function Assert-True { + param([bool]$Condition, [string]$Message) + if (-not $Condition) { + [void]$script:failures.Add("FAIL: $Message") + } +} + +function Write-PackageMetadata { + param([string]$VersionDirectory, [string]$Source) + New-Item -ItemType Directory -Path $VersionDirectory -Force | Out-Null + @{ version = 2; contentHash = 'test'; source = $Source } | + ConvertTo-Json | Set-Content -LiteralPath ( + Join-Path $VersionDirectory '.nupkg.metadata') -Encoding UTF8 +} + +try { + $packagesDirectory = Join-Path $tempRoot 'packages' + $localRepository = Join-Path $tempRoot 'feed' + New-Item -ItemType Directory -Path $localRepository -Force | Out-Null + + $localMachine = Join-Path $packagesDirectory 'sil.machine\3.9.2' + $publishedMachine = Join-Path $packagesDirectory 'sil.machine\3.9.3' + $unrelatedPackage = Join-Path $packagesDirectory 'example.package\1.0.0' + Write-PackageMetadata -VersionDirectory $localMachine -Source $localRepository + Write-PackageMetadata -VersionDirectory $publishedMachine ` + -Source 'https://api.nuget.org/v3/index.json' + Write-PackageMetadata -VersionDirectory $unrelatedPackage -Source $localRepository + + Set-Content -LiteralPath (Join-Path $localRepository 'SIL.Machine.3.9.2.nupkg') ` + -Value 'local package' + Set-Content -LiteralPath ( + Join-Path $localRepository 'SIL.Machine.Morphology.HermitCrab.3.9.2.snupkg') ` + -Value 'local symbols' + Set-Content -LiteralPath (Join-Path $localRepository 'Example.Package.1.0.0.nupkg') ` + -Value 'unrelated package' + $managedFeedPackages = @( + 'SIL.Core.18.0.0.nupkg', + 'SIL.LCModel.11.0.0.nupkg', + 'SIL.Chorus.LibChorus.6.0.0.nupkg', + 'L10NSharp.10.0.0.nupkg' + ) + foreach ($packageName in $managedFeedPackages) { + Set-Content -LiteralPath (Join-Path $localRepository $packageName) ` + -Value 'managed package' + } + + Import-Module (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Force + $config = Get-FieldWorksLocalLibraryConfig + Assert-True ($config.Keys.Count -eq 5) 'The catalogue should contain five libraries.' + foreach ($library in @('palaso', 'lcm', 'chorus', 'machine', 'l10nsharp')) { + Assert-True $config.Contains($library) "The catalogue should contain $library." + } + + Clear-FieldWorksLocalLibraries -PackagesDirectory $packagesDirectory ` + -LocalRepository $localRepository + + Assert-True (-not (Test-Path $localMachine)) ` + 'Cleanup should remove cache entries restored from a filesystem source.' + Assert-True (Test-Path $publishedMachine) ` + 'Cleanup should preserve cache entries restored from an HTTP source.' + Assert-True (Test-Path $unrelatedPackage) ` + 'Cleanup should preserve packages outside the managed library catalogue.' + Assert-True (-not (Test-Path ( + Join-Path $localRepository 'SIL.Machine.3.9.2.nupkg'))) ` + 'Cleanup should remove managed packages from the local feed.' + Assert-True (-not (Test-Path ( + Join-Path $localRepository 'SIL.Machine.Morphology.HermitCrab.3.9.2.snupkg'))) ` + 'Cleanup should remove managed symbol packages from the local feed.' + Assert-True (Test-Path (Join-Path $localRepository 'Example.Package.1.0.0.nupkg')) ` + 'Cleanup should preserve unrelated packages in the local feed.' + foreach ($packageName in $managedFeedPackages) { + Assert-True (-not (Test-Path (Join-Path $localRepository $packageName))) ` + "Cleanup should remove $packageName." + } + + Clear-FieldWorksLibraryPackageCache -PackagesDirectory $packagesDirectory ` + -Libraries @('machine') + Assert-True (-not (Test-Path $publishedMachine)) ` + 'Selected packing should evict a published cache entry with the same version.' + Assert-True (Test-Path $unrelatedPackage) ` + 'Selected packing should preserve cache entries outside its package family.' + + $managerText = Get-Content -LiteralPath ( + Join-Path $PSScriptRoot 'Manage-LocalLibraries.ps1') -Raw + Assert-True ($managerText -match '\$VersionOutputPath') ` + 'Manage-LocalLibraries should accept a packed-version output path.' + Assert-True ($managerText -match 'Import-Module.+LocalLibraries\.psm1') ` + 'Manage-LocalLibraries should import the shared library catalogue.' + Assert-True ($managerText -match 'Clear-FieldWorksLocalLibraries') ` + 'Manage-LocalLibraries should use the shared cache cleanup.' + Assert-True ($managerText -match 'Clear-FieldWorksLibraryPackageCache') ` + 'Pack mode should evict selected package families before local restore.' + Assert-True ($managerText -match 'Packing local libraries is build-scoped') ` + 'Direct pack mode should direct callers to build.ps1.' + Assert-True ($managerText -notmatch 'dotnet nuget add source') ` + 'Manage-LocalLibraries should not persist a user-level NuGet source.' + Assert-True ($managerText -match 'RestoreAdditionalProjectSources=\$LocalRepo') ` + 'Local pack restores should receive the local feed for dependent libraries.' + + $buildText = Get-Content -LiteralPath (Join-Path $PSScriptRoot '..\build.ps1') -Raw + Assert-True ($buildText -match '\[string\[\]\]\$LocalLibraries') ` + 'build.ps1 should accept a LocalLibraries array.' + Assert-True ($buildText -match 'Clear-FieldWorksLocalLibraries') ` + 'build.ps1 should clean unselected local libraries before restore.' + Assert-True ($buildText -match 'VersionOutputPath') ` + 'build.ps1 should consume non-persistent packed version output.' + Assert-True (($buildText -match 'Get-FieldWorksLocalFeedPath') -and ` + ($buildText -match 'RestoreAdditionalProjectSources')) ` + 'build.ps1 should add the resolved local feed to configured restore sources.' + Assert-True ($buildText -notmatch '\$env:LOCAL_NUGET_REPO') ` + 'build.ps1 should resolve the feed through the module, not the env var.' + + # --- Feed location ----------------------------------------------------- + # A machine-wide feed lets one working tree delete another's fresh packages. + $savedFeed = $env:LOCAL_NUGET_REPO + try { + $env:LOCAL_NUGET_REPO = $null + Assert-True ((Get-FieldWorksLocalFeedPath -RepositoryRoot 'C:\wt') -eq ` + 'C:\wt\.localfeed') ` + 'The feed should default inside the working tree.' + $env:LOCAL_NUGET_REPO = 'C:/legacy/feed' + Assert-True ((Get-FieldWorksLocalFeedPath -RepositoryRoot 'C:\wt') -eq ` + 'C:/legacy/feed') ` + 'An explicit LOCAL_NUGET_REPO should still win.' + } + finally { + $env:LOCAL_NUGET_REPO = $savedFeed + } + + # --- Version labels ---------------------------------------------------- + # SemVer pre-release identifiers allow only alphanumerics and hyphens. + Assert-True ((ConvertTo-FieldWorksVersionLabel -BranchName 'feature/x_y') -eq ` + 'feature-x-y') 'Branch separators should become hyphens.' + Assert-True ((ConvertTo-FieldWorksVersionLabel -BranchName 'LT-1/2') -eq ` + 'lt-1-2') 'Labels should be lowercased.' + Assert-True ((ConvertTo-FieldWorksVersionLabel ` + -BranchName ('a' * 60)).Length -le 24) 'Long branches should truncate.' + Assert-True ((ConvertTo-FieldWorksVersionLabel -BranchName '///') -eq ` + 'detached') 'A branch with no usable characters should be named.' + + # --- Pack versions --- + # A clean tree is identified by its commit, so its cache entry may be + # reused; a dirty tree has no stable identity. + $cleanState = [pscustomobject]@{ + Branch = 'feature/x'; Label = 'feature-x'; ShortSha = 'abc1234'; IsDirty = $false } + $dirtyState = [pscustomobject]@{ + Branch = 'feature/x'; Label = 'feature-x'; ShortSha = 'abc1234'; IsDirty = $true } + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '3.9.2' ` + -SourceState $cleanState) -eq '3.9.2-feature-x.abc1234') ` + 'A clean pack should be identified by branch and commit.' + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '11.0.0-beta0178' ` + -SourceState $cleanState) -eq '11.0.0-feature-x.abc1234') ` + 'A published pre-release label should be replaced, not appended to.' + # The core comes from the library's own GitVersion where it has one, so a + # version bump in the library shows up in the local package. + $config = Get-FieldWorksLocalLibraryConfig + Assert-True ($config['lcm'].VersionProject -eq 'src/SIL.LCModel/SIL.LCModel.csproj') ` + 'GitVersion libraries should name a project to read their version from.' + Assert-True (-not $config['machine'].Contains('VersionProject')) ` + 'A library without GitVersion should fall back to the consumed version.' + Assert-True ((Get-FieldWorksLibraryCoreVersion -SourceDirectory 'C: +ope' ` + -LibraryEntry $config['machine'] -FallbackVersion '3.9.2') -eq '3.9.2') ` + 'The fallback should be the consumed version core.' + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '3.9.2' ` + -SourceState $dirtyState) -eq '3.9.2-feature-x.dirty') ` + 'A dirty pack should be marked dirty rather than pinned to a commit.' + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '3.9.2' ` + -SourceState $cleanState) -ne '3.9.2') ` + 'A local pack must never reuse the published version string.' + + Assert-True ($managerText -match '-p:Version=\$packVersion') ` + 'Pack should stamp the derived version instead of sniffing filenames.' + Assert-True ($managerText -notmatch 'function Get-PackageVersion') ` + 'Pack should not parse versions out of filenames any more.' + # An earlier local pack is filesystem-sourced, so the pre-pack cleanup + # already removes it; clearing again would evict published packages too. + Assert-True (([regex]::Matches($managerText, + 'Clear-FieldWorksLibraryPackageCache')).Count -eq 1) ` + 'Only SetVersion mode should clear every cached version of a library.' + + # --- Pack reuse --- + # A clean commit identifies its contents, so an existing package for that + # version is the package this pack would produce. + $reuseFeed = Join-Path $tempRoot 'reusefeed' + New-Item -ItemType Directory -Force $reuseFeed | Out-Null + Set-Content -LiteralPath (Join-Path $reuseFeed 'SIL.Machine.3.9.2-b.abc1234.nupkg') ` + -Value 'x' + Assert-True (Test-FieldWorksPackIsCurrent -LocalRepository $reuseFeed ` + -PackVersion '3.9.2-b.abc1234' -SourceState $cleanState) ` + 'An existing package for a clean commit should be reused.' + Assert-True (-not (Test-FieldWorksPackIsCurrent -LocalRepository $reuseFeed ` + -PackVersion '3.9.2-b.abc1234' -SourceState $dirtyState)) ` + 'A dirty tree must never reuse an existing package.' + Assert-True (-not (Test-FieldWorksPackIsCurrent -LocalRepository $reuseFeed ` + -PackVersion '3.9.2-b.9999999' -SourceState $cleanState)) ` + 'A different commit should not be satisfied by another version.' + Assert-True ($managerText -match 'Reusing the packed') ` + 'Pack should report when it reuses an existing package.' + Assert-True ($managerText -match 'Uncommitted changes force a repack') ` + 'Pack should name the paths that force a repack.' + + # --- Worktree setup --- + # Branch names may contain path separators; the directory keeps the full + # name so a developer recognises which worktree is which. + Assert-True ((ConvertTo-FieldWorksWorktreeName -BranchName 'feature/x') -eq ` + 'feature-x') 'A branch name should become one directory name.' + Assert-True ((ConvertTo-FieldWorksWorktreeName ` + -BranchName 'LT-22728-keep-full-name') -eq 'LT-22728-keep-full-name') ` + 'A worktree name should not be truncated or lowercased.' + Assert-True ($config['lcm'].RepoDirectory -eq 'liblcm') ` + 'Each library should name its sibling checkout directory.' + + # --- One catalogue, three declarations --- + # The library names appear in the catalogue and in two ValidateSets, so + # each must agree with the catalogue. + $catalogue = @($config.Keys | Sort-Object) + foreach ($source in @(@{ n = 'build.ps1'; t = $buildText }, + @{ n = 'Manage-LocalLibraries.ps1'; t = $managerText })) { + $declared = [regex]::Matches($source.t, + "\[ValidateSet\((?'(?:palaso|lcm|chorus|machine|l10nsharp)'[^)]*)\)\]") + Assert-True ($declared.Count -ge 1) ` + "$($source.n) should declare the library names." + if ($declared.Count -ge 1) { + $names = @([regex]::Matches($declared[0].Groups['set'].Value, "'([^']+)'") | + ForEach-Object { $_.Groups[1].Value } | Sort-Object) + Assert-True (($names -join ',') -eq ($catalogue -join ',')) ` + "$($source.n) ValidateSet should match the catalogue exactly." + } + } + + # --- Odd cache entries --- + # Version 1 metadata records no source, and reading a missing property + # under Set-StrictMode is terminating. + $oddRoot = Join-Path $tempRoot 'odd' + $v1 = Join-Path $oddRoot 'packages/sil.lcmodel/1.0.0' + New-Item -ItemType Directory -Path $v1 -Force | Out-Null + '{ "version": 1, "contentHash": "abc" }' | + Set-Content -LiteralPath (Join-Path $v1 '.nupkg.metadata') -Encoding UTF8 + $partial = Join-Path $oddRoot 'packages/sil.lcmodel/2.0.0' + New-Item -ItemType Directory -Path $partial -Force | Out-Null + Clear-FieldWorksLocalLibraries -PackagesDirectory (Join-Path $oddRoot 'packages') ` + -LocalRepository (Join-Path $oddRoot 'feed') -Libraries @('lcm') + Assert-True (Test-Path -LiteralPath $v1) ` + 'Version 1 metadata should be preserved, not crash the build.' + Assert-True (-not (Test-Path -LiteralPath $partial)) ` + 'A version directory with no metadata is a partial extraction and should go.' + + # --- Reaching a restore this build does not launch --- + # A restore run through Exec is a new process, so an override has to be on + # disk where every process reads it, not on a command line. + $silVersions = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'SilVersions.props') -Raw + Assert-True ($silVersions -match 'LocalLibraries\.props') ` + 'SilVersions.props should import the generated local library overrides.' + Assert-True ($silVersions.IndexOf('LocalLibraries.props') -gt ` + $silVersions.IndexOf('SilLcmVersion')) ` + 'The override must be imported after the defaults so that it wins.' + $propsPath = Join-Path $tempRoot 'gen/Build/LocalLibraries.props' + Write-FieldWorksLocalLibraryProps -Path $propsPath ` + -Versions ([ordered]@{ SilLcmVersion = '9.9.9-x.abc1234' }) ` + -LocalRepository 'C:/feed' + [xml]$generated = Get-Content -LiteralPath $propsPath + Assert-True ($generated.Project.PropertyGroup.SilLcmVersion -eq '9.9.9-x.abc1234') ` + 'The generated overrides should carry the packed version.' + Assert-True ($generated.Project.PropertyGroup.RestoreAdditionalProjectSources ` + -match 'C:/feed') 'The generated overrides should carry the local feed.' + Remove-FieldWorksLocalLibraryProps -Path $propsPath + Assert-True (-not (Test-Path -LiteralPath $propsPath)) ` + 'A build selecting no library should remove the overrides.' + Assert-True ($buildText -match 'Remove-FieldWorksLocalLibraryProps') ` + 'Every build should clear a previous selection before restoring.' + + # --- Symbol directories --- + # A directory that does not exist copies nothing and says nothing, so the + # configured paths have to match where each library actually writes. + foreach ($name in $config.Keys) { + foreach ($relative in @($config[$name].PdbRelativeDir)) { + Assert-True ($relative -notmatch 'net462$' -or + $name -in @('palaso', 'lcm', 'chorus')) ` + "$name should not claim a net462 symbol directory." + } + } + Assert-True (@($config['machine'].PdbRelativeDir).Count -eq 2) ` + 'A library with per-project output needs one symbol directory per project.' + Assert-True ($config['l10nsharp'].PdbRelativeDir -eq 'output/Debug/net48') ` + 'L10NSharp symbols should come from a framework it actually builds.' + Assert-True ($managerText -match 'No PDB files found for') ` + 'A missing symbol directory should be reported, not passed over.' + + $testText = Get-Content -LiteralPath (Join-Path (Split-Path $PSScriptRoot -Parent) 'test.ps1') -Raw + Assert-True ($testText -match 'if \(-not \$TestProject -and -not \$TestFilter\)') ` + 'A targeted test run should not also run the local library tests.' + + $setupText = Get-Content -LiteralPath ` + (Join-Path $PSScriptRoot 'Setup-LocalLibraries.ps1') -Raw + Assert-True ($setupText -notmatch 'Read-Host|PromptForChoice|ReadLine') ` + 'Setup must fail with instructions rather than wait for input.' + $moduleText = Get-Content -LiteralPath ` + (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Raw + Assert-True (($moduleText + $setupText) -notmatch '&\s*git[^ + +]*(switch|checkout|reset|clean)') ` + 'Setup must never switch a branch in a checkout that already has one.' + Assert-True ($moduleText -match 'fetch --quiet') ` + 'Setup should fetch, which cannot disturb a working tree.' + Assert-True ($moduleText -notmatch 'git -C \$RepositoryPath pull') ` + 'Setup must not pull, which can conflict or lose work.' + Assert-True ($moduleText -match 'worktree list --porcelain') ` + 'An existing worktree for the branch should be discovered, not recreated.' + Assert-True ($moduleText -match '--path-format=absolute') ` + 'Git directory lookups must be absolute, not relative to the caller.' + + [xml]$nugetConfig = Get-Content -LiteralPath ( + Join-Path (Split-Path $PSScriptRoot -Parent) 'nuget.config') + # A local build adds its feed for that build only, so inherited + # sources can stay and a developer's private feed keeps working. + Assert-True ($null -eq $nugetConfig.SelectSingleNode( + '/configuration/packageSources/clear')) ` + 'nuget.config should leave inherited package sources in place.' +} +finally { + if (Test-Path $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force + } +} + +if ($failures.Count -gt 0) { + # Write-Host, not Write-Error: this script sets ErrorActionPreference to + # Stop, which would make the first failure terminating and hide the rest. + $failures | ForEach-Object { Write-Host $_ -ForegroundColor Red } + Write-Host ("{0} local library test(s) failed." -f $failures.Count) ` + -ForegroundColor Red + exit 1 +} + +Write-Host 'Local library tests passed.' -ForegroundColor Green diff --git a/Build/LocalLibraries.psm1 b/Build/LocalLibraries.psm1 new file mode 100644 index 0000000000..a451e533fe --- /dev/null +++ b/Build/LocalLibraries.psm1 @@ -0,0 +1,547 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Ordered, and libpalaso is first on purpose: packing order is significant +# because other libraries may depend on it. Do not alphabetise these. +$script:LibraryConfig = [ordered]@{ + palaso = @{ + VersionProperty = 'SilLibPalasoVersion' + PdbRelativeDir = 'output/Debug/net462' + CachePrefixes = @( + 'sil.core', 'sil.windows', 'sil.dblbundle', 'sil.writingsystems', + 'sil.dictionary', 'sil.lift', 'sil.lexicon', 'sil.archiving', + 'sil.media', 'sil.scripture', 'sil.testutilities' + ) + EnvVar = 'LIBPALASO_PATH' + RepoDirectory = 'libpalaso' + VersionProject = 'SIL.Core/SIL.Core.csproj' + } + l10nsharp = @{ + VersionProperty = 'L10NSharpVersion' + PdbRelativeDir = 'output/Debug/net48' + CachePrefixes = @('l10nsharp') + EnvVar = 'L10NSHARP_PATH' + RepoDirectory = 'l10nsharp' + VersionProject = 'src/L10NSharp/L10NSharp.csproj' + } + lcm = @{ + VersionProperty = 'SilLcmVersion' + PdbRelativeDir = 'artifacts/Debug/net462' + CachePrefixes = @('sil.lcmodel') + EnvVar = 'LIBLCM_PATH' + RepoDirectory = 'liblcm' + VersionProject = 'src/SIL.LCModel/SIL.LCModel.csproj' + } + chorus = @{ + VersionProperty = 'SilChorusVersion' + PdbRelativeDir = 'output/Debug/net462' + CachePrefixes = @('sil.chorus') + EnvVar = 'LIBCHORUS_PATH' + RepoDirectory = 'chorus' + VersionProject = 'src/Chorus/Chorus.csproj' + } + machine = @{ + VersionProperty = 'SilMachineVersion' + PdbRelativeDir = @( + 'src/SIL.Machine/bin/Debug/netstandard2.0', + 'src/SIL.Machine.Morphology.HermitCrab/bin/Debug/netstandard2.0' + ) + CachePrefixes = @('sil.machine') + EnvVar = 'SILMACHINE_PATH' + RepoDirectory = 'machine' + # Only the projects FieldWorks uses, which avoids the native CMake + # dependencies the rest of the repository pulls in. + PackProjects = @( + 'src/SIL.Machine/SIL.Machine.csproj', + 'src/SIL.Machine.Morphology.HermitCrab/SIL.Machine.Morphology.HermitCrab.csproj' + ) + } +} + +<# +.SYNOPSIS + Returns the local NuGet feed directory for this working tree. +.DESCRIPTION + A machine-wide feed lets one working tree's build delete packages another + working tree just produced, so the feed defaults inside the repository. + LOCAL_NUGET_REPO still wins when set, for existing setups. +#> +function Get-FieldWorksLocalFeedPath { + param([string]$RepositoryRoot) + if (-not [string]::IsNullOrWhiteSpace($env:LOCAL_NUGET_REPO)) { + return $env:LOCAL_NUGET_REPO + } + return (Join-Path $RepositoryRoot '.localfeed') +} + +<# +.SYNOPSIS + Converts a git branch name into a NuGet pre-release label. +.DESCRIPTION + SemVer pre-release identifiers allow only ASCII alphanumerics and hyphens, + so branch separators such as '/' and '_' are replaced. The result is + truncated because older tooling limited the whole pre-release string. +#> +function ConvertTo-FieldWorksVersionLabel { + param([string]$BranchName) + $label = ($BranchName -replace '[^0-9A-Za-z-]', '-').Trim('-') + $label = $label -replace '-{2,}', '-' + if ([string]::IsNullOrWhiteSpace($label)) { + return 'detached' + } + if ($label.Length -gt 24) { + $label = $label.Substring(0, 24).Trim('-') + } + return $label.ToLowerInvariant() +} + +<# +.SYNOPSIS + Describes the git state of a local library checkout. +.DESCRIPTION + Returns the branch label, short commit and whether the tree is dirty. A + dirty tree cannot be identified by commit, so callers must repack instead of + trusting the version string. Untracked files count as dirty because a new + source file changes the build without changing the commit. +#> +function Get-FieldWorksLibrarySourceState { + param([string]$SourceDirectory) + + $branch = (& git -C $SourceDirectory rev-parse --abbrev-ref HEAD 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "'$SourceDirectory' is not a git checkout; cannot derive a local version." + } + $branch = "$branch".Trim() + if ($branch -eq 'HEAD') { + $branch = 'detached' + } + + $shortSha = "$(& git -C $SourceDirectory rev-parse --short=7 HEAD 2>$null)".Trim() + $status = @(& git -C $SourceDirectory status --porcelain --untracked-files=normal 2>$null) + $isDirty = $status.Count -gt 0 + + return [pscustomobject]@{ + Branch = $branch + Label = ConvertTo-FieldWorksVersionLabel -BranchName $branch + ShortSha = $shortSha + IsDirty = $isDirty + DirtyPaths = @($status | ForEach-Object { ($_ -replace '^.{2,3}', '').Trim() }) + } +} + +<# +.SYNOPSIS + Reports whether the feed already holds this exact packed version. +.DESCRIPTION + A version derived from a clean commit identifies its contents, so an + existing package for it is the same package this pack would produce. A + dirty tree reuses one version string for changing contents, so its package + can never stand in. +#> +function Test-FieldWorksPackIsCurrent { + param([string]$LocalRepository, [string]$PackVersion, [pscustomobject]$SourceState) + if ($SourceState.IsDirty) { + return $false + } + if (-not (Test-Path -LiteralPath $LocalRepository)) { + return $false + } + return @(Get-ChildItem -LiteralPath $LocalRepository ` + -Filter "*.$PackVersion.nupkg" -File -ErrorAction SilentlyContinue).Count -gt 0 +} + +<# +.SYNOPSIS + Returns the Major.Minor.Patch a library gives itself. +.DESCRIPTION + Read from the library's own GitVersion so a version bump in the library is + reflected in the local package. Libraries without GitVersion have no + VersionProject and fall back to the version FieldWorks consumes. +#> +function Get-FieldWorksLibraryCoreVersion { + param([string]$SourceDirectory, [hashtable]$LibraryEntry, [string]$FallbackVersion) + + $fallbackCore = ($FallbackVersion -split '-', 2)[0] + if (-not $LibraryEntry.Contains('VersionProject')) { + return $fallbackCore + } + $project = Join-Path $SourceDirectory $LibraryEntry.VersionProject + if (-not (Test-Path -LiteralPath $project)) { + Write-Warning ("Version project '$project' not found; using $fallbackCore.") + return $fallbackCore + } + + # -restore first: GetVersion comes from the GitVersion package, which a + # checkout that has never been built does not have yet. + $probed = & dotnet msbuild $project -restore -t:GetVersion ` + -getProperty:GitVersion_MajorMinorPatch -v:q -nologo 2>$null + $probed = @($probed | Where-Object { $_ -match '^\d+\.\d+\.\d+$' }) + if ($LASTEXITCODE -ne 0 -or $probed.Count -eq 0) { + Write-Warning ("Could not read a GitVersion version from '$project'; " + + "using $fallbackCore.") + return $fallbackCore + } + return $probed[-1].Trim() +} + +<# +.SYNOPSIS + Builds the pre-release version for a locally packed library. +.DESCRIPTION + A clean tree is identified by its commit, so repacking the same commit + reuses the cached package correctly. A dirty tree has no stable identity, + so it is labelled 'dirty' and the caller repacks every time. +#> +function Get-FieldWorksLocalPackVersion { + param([string]$CoreVersion, [pscustomobject]$SourceState) + + $core = ($CoreVersion -split '-', 2)[0] + if ($SourceState.IsDirty) { + return "$core-$($SourceState.Label).dirty" + } + return "$core-$($SourceState.Label).$($SourceState.ShortSha)" +} + +function Test-ManagedPackageName { + param([string]$Name, [string[]]$Prefixes) + $normalizedName = $Name.ToLowerInvariant() + foreach ($prefix in $Prefixes) { + if ($normalizedName -eq $prefix -or $normalizedName.StartsWith("$prefix.")) { + return $true + } + } + return $false +} + +function Test-FilesystemPackageSource { + param([string]$Source) + if ([string]::IsNullOrWhiteSpace($Source)) { + return $false + } + $uri = $null + if ([System.Uri]::TryCreate($Source, [System.UriKind]::Absolute, [ref]$uri)) { + return $uri.IsFile + } + return [System.IO.Path]::IsPathRooted($Source) +} + +function Get-SelectedPrefixes { + param([string[]]$Libraries) + $selected = if ($Libraries -and $Libraries.Count -gt 0) { + $Libraries + } + else { + @($script:LibraryConfig.Keys) + } + $prefixes = foreach ($library in $selected) { + if (-not $script:LibraryConfig.Contains($library)) { + throw "Unknown local library '$library'." + } + $script:LibraryConfig[$library].CachePrefixes + } + return @($prefixes | Sort-Object -Unique) +} + +<# +.SYNOPSIS + Removes every cached version for the selected local-library groups. +#> +function Clear-FieldWorksLibraryPackageCache { + param([string]$PackagesDirectory, [string[]]$Libraries) + if (-not (Test-Path -LiteralPath $PackagesDirectory)) { + return + } + $prefixes = Get-SelectedPrefixes -Libraries $Libraries + $packageDirectories = @(Get-ChildItem -LiteralPath $PackagesDirectory -Directory | + Where-Object { Test-ManagedPackageName -Name $_.Name -Prefixes $prefixes }) + foreach ($packageDirectory in $packageDirectories) { + Remove-Item -LiteralPath $packageDirectory.FullName -Recurse -Force + } + if ($packageDirectories.Count -gt 0) { + Write-Host ("Cleared {0} package cache folders." -f $packageDirectories.Count) ` + -ForegroundColor Yellow + } +} + +<# +.SYNOPSIS + Returns the configuration for FieldWorks-supported local libraries. +#> +function Get-FieldWorksLocalLibraryConfig { + return $script:LibraryConfig +} + +<# +.SYNOPSIS + Removes locally sourced cache entries and managed packages from a local feed. +#> +function Clear-FieldWorksLocalLibraries { + param( + [string]$PackagesDirectory, + [string]$LocalRepository, + [string[]]$Libraries + ) + + $prefixes = Get-SelectedPrefixes -Libraries $Libraries + $cacheRemovalCount = 0 + $feedRemovalCount = 0 + + if (Test-Path -LiteralPath $PackagesDirectory) { + $packageDirectories = @(Get-ChildItem -LiteralPath $PackagesDirectory -Directory | + Where-Object { Test-ManagedPackageName -Name $_.Name -Prefixes $prefixes }) + foreach ($packageDirectory in $packageDirectories) { + foreach ($versionDirectory in @(Get-ChildItem -LiteralPath $packageDirectory.FullName -Directory)) { + $metadataPath = Join-Path $versionDirectory.FullName '.nupkg.metadata' + if (-not (Test-Path -LiteralPath $metadataPath)) { + # No metadata means extraction never finished, so nothing here can + # be trusted and restore will replace it. + Remove-Item -LiteralPath $versionDirectory.FullName -Recurse -Force + $cacheRemovalCount++ + continue + } + $source = $null + try { + $metadata = Get-Content -LiteralPath $metadataPath -Raw | ConvertFrom-Json + # Version 1 metadata records no source; reading it under + # Set-StrictMode is a terminating error, so it stays guarded. + $source = $metadata.PSObject.Properties['source'] + if ($source) { $source = [string]$source.Value } + } + catch { + Write-Warning "Could not read NuGet metadata at '$metadataPath'; preserving it." + continue + } + if (Test-FilesystemPackageSource -Source $source) { + Remove-Item -LiteralPath $versionDirectory.FullName -Recurse -Force + $cacheRemovalCount++ + } + } + if (@(Get-ChildItem -LiteralPath $packageDirectory.FullName -Force).Count -eq 0) { + Remove-Item -LiteralPath $packageDirectory.FullName -Force + } + } + } + + if ($LocalRepository -and (Test-Path -LiteralPath $LocalRepository)) { + $feedPackages = @(Get-ChildItem -LiteralPath $LocalRepository -File | + Where-Object { + $_.Extension -in @('.nupkg', '.snupkg') -and + (Test-ManagedPackageName -Name $_.BaseName -Prefixes $prefixes) + }) + foreach ($feedPackage in $feedPackages) { + Remove-Item -LiteralPath $feedPackage.FullName -Force + $feedRemovalCount++ + } + } + + if ($cacheRemovalCount -gt 0 -or $feedRemovalCount -gt 0) { + Write-Host ("Cleared {0} local cache entries and {1} local feed packages." -f ` + $cacheRemovalCount, $feedRemovalCount) -ForegroundColor Yellow + } +} + +<# +.SYNOPSIS + Converts a branch name into a directory name. +.DESCRIPTION + Branch names may contain path separators, which cannot appear in a single + directory name. Unlike a version label this keeps the full name, because + the directory is how a developer recognises the worktree. +#> +function ConvertTo-FieldWorksWorktreeName { + param([string]$BranchName) + $name = $BranchName + foreach ($invalid in [System.IO.Path]::GetInvalidFileNameChars()) { + $name = $name.Replace($invalid, '-') + } + return ($name -replace '-{2,}', '-').Trim('-') +} + +<# +.SYNOPSIS + Locates a local checkout of a library. +.DESCRIPTION + Prefers an explicit path, then a sibling of the FieldWorks checkout, then + the legacy environment variable. Throws with the ways to fix it rather than + prompting, so unattended builds fail instead of waiting for input. +#> +function Resolve-FieldWorksLibraryRepo { + param([string]$Library, [string]$RepositoryRoot, [string]$ExplicitPath) + + $entry = $script:LibraryConfig[$Library] + if (-not $entry) { + throw "Unknown local library '$Library'." + } + + # Siblings sit beside the main checkout, which is not the parent of a + # worktree, so ask git where the repository itself lives. + $mainRoot = $RepositoryRoot + $commonDir = & git -C $RepositoryRoot rev-parse --path-format=absolute ` + --git-common-dir 2>$null + if ($LASTEXITCODE -eq 0 -and $commonDir) { + $mainRoot = Split-Path ($commonDir.Trim()) -Parent + } + + $candidates = [ordered]@{} + if ($ExplicitPath) { $candidates['the path given'] = $ExplicitPath } + $sibling = Join-Path (Split-Path $mainRoot -Parent) $entry.RepoDirectory + $candidates["the sibling checkout $sibling"] = $sibling + $fromEnv = [System.Environment]::GetEnvironmentVariable($entry.EnvVar) + if ($fromEnv) { $candidates["$($entry.EnvVar)"] = $fromEnv } + + foreach ($source in $candidates.Keys) { + $path = $candidates[$source] + if (Test-Path -LiteralPath (Join-Path $path '.git')) { + return [pscustomobject]@{ Path = (Resolve-Path -LiteralPath $path).Path + Source = $source } + } + } + + throw ("No git checkout of '$Library' was found. Clone it beside FieldWorks " + + "as '$sibling', or set $($entry.EnvVar), or pass an explicit path.") +} + +<# +.SYNOPSIS + Makes a worktree of a library branch available, or explains what is missing. +.DESCRIPTION + An existing worktree on the branch is used as it stands, because git refuses + to check one branch out twice and because another worktree may hold work in + progress. Otherwise a worktree is created under .tmp/worktrees. The branch is + never switched in a checkout that already has one, so no work can be lost. +#> +function Resolve-FieldWorksLibraryWorktree { + param([string]$RepositoryPath, [string]$Branch, [switch]$Create) + + & git -C $RepositoryPath fetch --quiet 2>$null | Out-Null + + # Discovery, not creation: the branch may already be checked out somewhere, + # which is both the fast path and the only path git allows. + $listing = @(& git -C $RepositoryPath worktree list --porcelain 2>$null) + $currentPath = $null + foreach ($line in $listing) { + if ($line -like 'worktree *') { $currentPath = $line.Substring(9).Trim() } + elseif ($line -eq "branch refs/heads/$Branch") { + return [pscustomobject]@{ Path = $currentPath; Created = $false } + } + } + + $hasLocal = & git -C $RepositoryPath rev-parse --verify --quiet "refs/heads/$Branch" + $hasRemote = & git -C $RepositoryPath rev-parse --verify --quiet "refs/remotes/origin/$Branch" + if (-not $hasLocal -and -not $hasRemote) { + throw ("Branch '$Branch' does not exist in $RepositoryPath, locally or on " + + "origin. Create it there first, or name an existing branch.") + } + + $target = Join-Path $RepositoryPath (Join-Path '.tmp/worktrees' ` + (ConvertTo-FieldWorksWorktreeName -BranchName $Branch)) + if (-not $Create) { + throw ("Branch '$Branch' has no worktree in $RepositoryPath. Run with " + + "-SetupLocalLibraries to create one at $target.") + } + + # Their .gitignore belongs to another team, so exclude .tmp for this clone + # only. --git-common-dir is relative to the caller unless asked otherwise. + $commonDir = & git -C $RepositoryPath rev-parse --path-format=absolute ` + --git-common-dir 2>$null + if ($LASTEXITCODE -eq 0 -and $commonDir) { + $excludePath = Join-Path $commonDir.Trim() 'info/exclude' + $excluded = if (Test-Path -LiteralPath $excludePath) { + @(Get-Content -LiteralPath $excludePath) + } else { @() } + if (-not @($excluded | Where-Object { $_.Trim() -eq '.tmp/' -or + $_.Trim() -eq '.tmp/worktrees/' })) { + Add-Content -LiteralPath $excludePath -Value '.tmp/' + Write-Host " Excluded .tmp/ for this clone only." -ForegroundColor Gray + } + } + + $addArgs = if ($hasLocal) { @($target, $Branch) } + else { @('-b', $Branch, $target, "origin/$Branch") } + & git -C $RepositoryPath worktree add @addArgs 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $target)) { + throw "Could not create a worktree for '$Branch' at $target." + } + return [pscustomobject]@{ Path = $target; Created = $true } +} + +<# +.SYNOPSIS + Describes how a checkout stands against its upstream branch. +#> +function Get-FieldWorksLibraryUpstreamState { + param([string]$WorktreePath) + $counts = & git -C $WorktreePath rev-list --left-right --count '@{upstream}...HEAD' 2>$null + if ($LASTEXITCODE -ne 0 -or -not $counts) { + return [pscustomobject]@{ HasUpstream = $false; Behind = 0; Ahead = 0 } + } + $parts = -split $counts + return [pscustomobject]@{ HasUpstream = $true + Behind = [int]$parts[0]; Ahead = [int]$parts[1] } +} + +<# +.SYNOPSIS + Writes the generated property file that selects local libraries. +.DESCRIPTION + Written to disk rather than passed on a command line because a restore + launched through Exec starts a new MSBuild process, which does not inherit + global properties. SilVersions.props imports this file, so every process + that resolves a version sees the same answer. +#> +function Write-FieldWorksLocalLibraryProps { + param([string]$Path, [hashtable]$Versions, [string]$LocalRepository) + + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add('') + $lines.Add("`t") + $lines.Add("`t") + foreach ($name in $Versions.Keys) { + $lines.Add("`t`t<$name>$($Versions[$name])") + } + if ($LocalRepository) { + $lines.Add("`t`t" + + "`$(RestoreAdditionalProjectSources);$LocalRepository" + + '') + } + $lines.Add("`t") + $lines.Add('') + + $directory = Split-Path $Path -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + Set-Content -LiteralPath $Path -Value $lines -Encoding UTF8 +} + +<# +.SYNOPSIS + Removes the generated local library property file, if it is present. +#> +function Remove-FieldWorksLocalLibraryProps { + param([string]$Path) + if (Test-Path -LiteralPath $Path) { + Remove-Item -LiteralPath $Path -Force + Write-Host 'Removed local library overrides; using published versions.' ` + -ForegroundColor Yellow + } +} + +<# +.SYNOPSIS + Returns the path of the generated local library property file. +#> +function Get-FieldWorksLocalLibraryPropsPath { + param([string]$RepositoryRoot) + return (Join-Path $RepositoryRoot 'Build/LocalLibraries.props') +} + +Export-ModuleMember -Function Get-FieldWorksLocalLibraryConfig, + Clear-FieldWorksLocalLibraries, Clear-FieldWorksLibraryPackageCache, + Get-FieldWorksLocalFeedPath, ConvertTo-FieldWorksVersionLabel, + Get-FieldWorksLibrarySourceState, Get-FieldWorksLocalPackVersion, + Get-FieldWorksLibraryCoreVersion, Test-FieldWorksPackIsCurrent, + ConvertTo-FieldWorksWorktreeName, Resolve-FieldWorksLibraryRepo, + Resolve-FieldWorksLibraryWorktree, Get-FieldWorksLibraryUpstreamState, + Write-FieldWorksLocalLibraryProps, Remove-FieldWorksLocalLibraryProps, + Get-FieldWorksLocalLibraryPropsPath diff --git a/Build/Manage-LocalLibraries.ps1 b/Build/Manage-LocalLibraries.ps1 index 2f5e7eb1e5..eeb6e89ce2 100644 --- a/Build/Manage-LocalLibraries.ps1 +++ b/Build/Manage-LocalLibraries.ps1 @@ -5,11 +5,11 @@ .DESCRIPTION Two modes of operation: - Pack mode (one or more source paths provided): - Packs local checkouts of liblcm, libpalaso, chorus, and/or machine into the - local NuGet feed using each library's own version. Detects the version - from produced packages, updates SilVersions.props to match, copies - PDBs, and clears stale cached packages. + Build pack mode (one or more source paths and -VersionOutputPath provided): + Packs local checkouts of liblcm, libpalaso, chorus, machine, and/or + L10NSharp into the local NuGet feed using each library's own version. + Detects the versions, writes them for build.ps1, copies PDBs, and clears + stale cached packages. Multiple libraries can be packed in a single call. libpalaso is always packed first (other libraries may depend on it). @@ -66,13 +66,16 @@ Sets the version in SilVersions.props (SetVersion mode). Use to revert to an upstream version. Not used in pack mode. +.PARAMETER VersionOutputPath + Path to write the packed versions to, as JSON keyed by version property. + .EXAMPLE - .\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso - Packs libpalaso, detects its version, and updates SilVersions.props. + .\build.ps1 -LocalLibraries palaso + Rebuilds libpalaso from LIBPALASO_PATH for this FieldWorks build. .EXAMPLE - .\Build\Manage-LocalLibraries.ps1 -Palaso -Chorus -ChorusPath C:\Repos\chorus - Packs libpalaso (from env var) and then chorus from the given path. + .\build.ps1 -LocalLibraries palaso,chorus + Rebuilds libpalaso and chorus from their configured paths for this build. .EXAMPLE .\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0 @@ -98,59 +101,20 @@ param( [ValidateSet('palaso', 'lcm', 'chorus', 'machine', 'l10nsharp')] [string]$Library, - [string]$Version -) + [string]$Version, -$ErrorActionPreference = "Stop" - -# --------------------------------------------------------------------------- -# Library-specific configuration -# --------------------------------------------------------------------------- + [string]$VersionOutputPath, -$LibraryConfig = @{ - palaso = @{ - VersionProperty = 'SilLibPalasoVersion' - PdbRelativeDir = 'output/Debug/net462' - CachePrefixes = @( - 'sil.core', 'sil.windows', 'sil.dblbundle', 'sil.writingsystems', - 'sil.dictionary', 'sil.lift', 'sil.lexicon', 'sil.archiving', - 'sil.media', 'sil.scripture', 'sil.testutilities' - ) - EnvVar = 'LIBPALASO_PATH' - } - lcm = @{ - VersionProperty = 'SilLcmVersion' - PdbRelativeDir = 'artifacts/Debug/net462' - CachePrefixes = @('sil.lcmodel') - EnvVar = 'LIBLCM_PATH' - } - chorus = @{ - VersionProperty = 'SilChorusVersion' - PdbRelativeDir = 'output/Debug/net462' - CachePrefixes = @('sil.chorus') - EnvVar = 'LIBCHORUS_PATH' - } - machine = @{ - VersionProperty = 'SilMachineVersion' - PdbRelativeDir = 'bin/Debug/netstandard2.0' - CachePrefixes = @('sil.machine') - EnvVar = 'SILMACHINE_PATH' - # Pack only the projects FieldWorks uses (avoids native CMake deps) - PackProjects = @( - 'src/SIL.Machine/SIL.Machine.csproj', - 'src/SIL.Machine.Morphology.HermitCrab/SIL.Machine.Morphology.HermitCrab.csproj' - ) - } - l10nsharp = @{ - VersionProperty = 'L10NSharpVersion' - PdbRelativeDir = 'output/Debug/net462' - CachePrefixes = @('l10nsharp') - EnvVar = 'L10NSHARP_PATH' - } -} + [string]$LocalFeedPath +) -# Pack order: libpalaso first (other libraries may depend on it) -$PackOrder = @('palaso', 'l10nsharp', 'lcm', 'chorus', 'machine') +$ErrorActionPreference = "Stop" +Import-Module (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Force +$LibraryConfig = Get-FieldWorksLocalLibraryConfig +# Pack order: libpalaso first, because other libraries may depend on it. The +# order comes from the catalogue's declaration order, so do not alphabetise it. +$PackOrder = @($LibraryConfig.Keys) +$packedVersions = [ordered]@{} # --------------------------------------------------------------------------- # Read SilVersions.props @@ -205,34 +169,49 @@ function Update-VersionAndClearCache { Write-Host "Updated SilVersions.props ($($cfg.VersionProperty) = $NewVersion)" -ForegroundColor Yellow - $packagesDir = Join-Path $repoRoot "packages" - if (Test-Path $packagesDir) { - $patterns = $cfg.CachePrefixes | ForEach-Object { "$packagesDir/$_*" } - $stale = @(Get-ChildItem -Path $patterns -Directory -ErrorAction SilentlyContinue) - if ($stale.Count -gt 0) { - $stale | Remove-Item -Recurse -Force - Write-Host "Cleared $($stale.Count) stale package folder(s) from packages/." -ForegroundColor Yellow - } - } + Clear-FieldWorksLibraryPackageCache -PackagesDirectory (Join-Path $repoRoot 'packages') ` + -Libraries @($LibName) } -# --------------------------------------------------------------------------- -# Helper: extract version from a .nupkg filename -# --------------------------------------------------------------------------- -# Split on '.', find the first segment starting with a digit — everything -# from there onward (minus .nupkg) is the version. -# E.g. SIL.Windows.Forms.Keyboarding.18.0.0-beta.nupkg → 18.0.0-beta - -function Get-PackageVersion { - param([string]$FileName) - $base = $FileName -replace '\.nupkg$', '' - $segments = $base -split '\.' - for ($i = 0; $i -lt $segments.Count; $i++) { - if ($segments[$i] -match '^\d') { - return ($segments[$i..($segments.Count - 1)] -join '.') +# --- Copy a library's PDBs next to the build output --- + +function Copy-LibrarySymbols { + param([string]$LibName, [string]$SourceDir) + $cfg = $LibraryConfig[$LibName] + + $outputDebugDir = Join-Path $repoRoot "Output/Debug" + $downloadsDir = Join-Path $repoRoot "Downloads" + $copied = 0 + + # A library that writes per-project output needs one directory per project. + foreach ($relativeDir in @($cfg.PdbRelativeDir)) { + $pdbSourceDir = Join-Path $SourceDir $relativeDir + if (-not (Test-Path $pdbSourceDir)) { + continue + } + $pdbFiles = @(Get-ChildItem -Path $pdbSourceDir -Filter "*.pdb" -File) + if ($pdbFiles.Count -eq 0) { + continue + } + foreach ($dir in @($outputDebugDir, $downloadsDir)) { + if (-not (Test-Path $dir)) { + New-Item -Path $dir -ItemType Directory -Force | Out-Null + } } + $pdbFiles | Copy-Item -Destination $outputDebugDir -Force + $pdbFiles | Copy-Item -Destination $downloadsDir -Force + $copied += $pdbFiles.Count + } + + if ($copied -gt 0) { + Write-Host "Copied $copied PDB file(s) to Output/Debug/ and Downloads/..." ` + -ForegroundColor Cyan + } + else { + # Say where we looked: a wrong directory here fails silently otherwise. + Write-Host ("No PDB files found for {0} under: {1}" -f $LibName, + ((@($cfg.PdbRelativeDir)) -join ', ')) -ForegroundColor Yellow } - return $null } # --------------------------------------------------------------------------- @@ -245,109 +224,108 @@ function Invoke-PackLibrary { $cfg = $LibraryConfig[$LibName] $node = Get-VersionNode $LibName + $packTargets = if ($cfg.PackProjects -and $cfg.PackProjects.Count -gt 0) { + @($cfg.PackProjects | ForEach-Object { Join-Path $SourceDir $_ }) + } else { @($SourceDir) } + + # NuGet resolves an extracted (id, version) before consulting a folder feed, + # so a local pack must never reuse the published version string. + $sourceState = Get-FieldWorksLibrarySourceState -SourceDirectory $SourceDir + $coreVersion = Get-FieldWorksLibraryCoreVersion -SourceDirectory $SourceDir ` + -LibraryEntry $cfg -FallbackVersion $node.InnerText.Trim() + $packVersion = Get-FieldWorksLocalPackVersion ` + -CoreVersion $coreVersion -SourceState $sourceState + Write-Host "" Write-Host "========================================" -ForegroundColor Cyan Write-Host "Packing $LibName" -ForegroundColor Cyan Write-Host " Source: $SourceDir" -ForegroundColor Cyan + $dirtyNote = if ($sourceState.IsDirty) { ' (dirty)' } else { '' } + Write-Host " Branch: $($sourceState.Branch)$dirtyNote" -ForegroundColor Cyan + Write-Host " Commit: $($sourceState.ShortSha)" -ForegroundColor Cyan Write-Host " Current: $($node.InnerText.Trim())" -ForegroundColor Cyan + Write-Host " Packing as: $packVersion" -ForegroundColor Cyan Write-Host " Output: $LocalRepo" -ForegroundColor Cyan Write-Host "========================================" -ForegroundColor Cyan - # Record timestamp before pack so we can find newly-produced packages - $packStart = Get-Date + if ($sourceState.IsDirty) { + # Name the paths: while the tree is dirty every build repacks, so the + # way to a reusable package is to commit or remove these. + $shown = @($sourceState.DirtyPaths | Select-Object -First 5) + Write-Host ("Uncommitted changes force a repack ({0}):" -f + $sourceState.DirtyPaths.Count) -ForegroundColor Yellow + $shown | ForEach-Object { Write-Host " $_" -ForegroundColor Yellow } + if ($sourceState.DirtyPaths.Count -gt $shown.Count) { + Write-Host (" ... and {0} more" -f + ($sourceState.DirtyPaths.Count - $shown.Count)) -ForegroundColor Yellow + } + } + elseif (Test-FieldWorksPackIsCurrent -LocalRepository $LocalRepo ` + -PackVersion $packVersion -SourceState $sourceState) { + Write-Host "Reusing the packed $packVersion (commit unchanged)." ` + -ForegroundColor Green + $script:packedVersions[$cfg.VersionProperty] = $packVersion + Copy-LibrarySymbols -LibName $LibName -SourceDir $SourceDir + return + } + + # Only the library being repacked loses its local packages. + Clear-FieldWorksLocalLibraries -PackagesDirectory (Join-Path $repoRoot 'packages') ` + -LocalRepository $LocalRepo -Libraries @($LibName) + + # Build first: a package may include output from a framework its own project + # does not build, which pack alone does not produce. + $commonBuildArgs = @( + '-c', 'Debug' + "-p:Version=$packVersion" + '-p:DisableGitVersionTask=true' + "-p:RestoreAdditionalProjectSources=$LocalRepo" + ) + Write-Host "Running dotnet build..." -ForegroundColor Cyan + foreach ($buildTarget in $packTargets) { + & dotnet build $buildTarget @commonBuildArgs + if ($LASTEXITCODE -ne 0) { + throw "dotnet build failed for $LibName ($buildTarget)." + } + } Write-Host "Running dotnet pack..." -ForegroundColor Cyan $commonPackArgs = @( '-c', 'Debug' + "-p:Version=$packVersion" + # GitVersion.MsBuild assigns Version inside a target, which outranks a + # command-line property, so it must be switched off for the stamp to hold. + '-p:DisableGitVersionTask=true' "-p:IncludeSymbols=true" "-p:SymbolPackageFormat=snupkg" + "-p:RestoreAdditionalProjectSources=$LocalRepo" '--output', $LocalRepo ) - $projects = $cfg.PackProjects - if ($projects -and $projects.Count -gt 0) { - foreach ($proj in $projects) { - $projPath = Join-Path $SourceDir $proj - & dotnet pack $projPath @commonPackArgs - if ($LASTEXITCODE -ne 0) { - throw "dotnet pack failed for $LibName ($proj)." - } - } - } - else { - & dotnet pack $SourceDir @commonPackArgs + foreach ($packTarget in $packTargets) { + & dotnet pack $packTarget @commonPackArgs if ($LASTEXITCODE -ne 0) { - throw "dotnet pack failed for $LibName." + throw "dotnet pack failed for $LibName ($packTarget)." } } - # Find .nupkg files created after pack started (exclude .snupkg and test pkgs) - $newPackages = @( - Get-ChildItem -Path $LocalRepo -Filter "*.nupkg" -File | - Where-Object { $_.LastWriteTime -ge $packStart -and $_.Extension -eq '.nupkg' -and $_.Name -notmatch 'tests' } + # Verify the stamped version actually reached the feed. + $produced = @( + Get-ChildItem -Path $LocalRepo -Filter "*.$packVersion.nupkg" -File | + Where-Object { $_.Name -notmatch 'tests' } ) - - if ($newPackages.Count -eq 0) { - $currentVer = (Get-VersionNode $LibName).InnerText.Trim() - Write-Host "" - Write-Host "WARNING: No new .nupkg files were produced for $LibName." -ForegroundColor Yellow - Write-Host " The library version may not have changed since the last pack." -ForegroundColor Yellow - Write-Host " Current version in SilVersions.props: $currentVer" -ForegroundColor Yellow - Write-Host " Skipping version update for $LibName." -ForegroundColor Yellow - return - } - - Write-Host "New packages found:" -ForegroundColor Gray - $newPackages | ForEach-Object { Write-Host " $($_.Name)" -ForegroundColor Gray } - - $detectedVersions = @($newPackages | ForEach-Object { Get-PackageVersion $_.Name } | - Where-Object { $_ } | Sort-Object -Unique) - - Write-Host "Detected version(s): $($detectedVersions -join ', ')" -ForegroundColor Gray - - if ($detectedVersions.Count -eq 0) { - throw "Could not parse version from produced packages: $($newPackages.Name -join ', ')" - } - if ($detectedVersions.Count -gt 1) { - Write-Host "WARNING: Multiple versions detected in produced packages:" -ForegroundColor Red - $detectedVersions | ForEach-Object { Write-Host " $_" -ForegroundColor Red } - throw "Expected all packages to share one version. Clean $LocalRepo and retry." + if ($produced.Count -eq 0) { + throw ("dotnet pack produced no package for $LibName at version " + + "$packVersion. Inspect $LocalRepo.") } - $packVersion = $detectedVersions[0] Write-Host "" - Write-Host "Pack complete ($($newPackages.Count) package(s), version $packVersion)." -ForegroundColor Green + Write-Host "Pack complete ($($produced.Count) package(s), version $packVersion)." ` + -ForegroundColor Green - # Update SilVersions.props and clear cache - Update-VersionAndClearCache -LibName $LibName -NewVersion $packVersion - Write-Host "To revert: git checkout Build/SilVersions.props" -ForegroundColor Yellow + $script:packedVersions[$cfg.VersionProperty] = $packVersion - # Copy PDB files to Output/Debug/ and Downloads/ - $pdbSourceDir = Join-Path $SourceDir $cfg.PdbRelativeDir - - if (Test-Path $pdbSourceDir) { - $outputDebugDir = Join-Path $repoRoot "Output/Debug" - $downloadsDir = Join-Path $repoRoot "Downloads" - - foreach ($dir in @($outputDebugDir, $downloadsDir)) { - if (-not (Test-Path $dir)) { - New-Item -Path $dir -ItemType Directory -Force | Out-Null - } - } - - $pdbFiles = @(Get-ChildItem -Path $pdbSourceDir -Filter "*.pdb" -File) - if ($pdbFiles.Count -gt 0) { - Write-Host "Copying $($pdbFiles.Count) PDB file(s) to Output/Debug/ and Downloads/..." -ForegroundColor Cyan - $pdbFiles | Copy-Item -Destination $outputDebugDir -Force - $pdbFiles | Copy-Item -Destination $downloadsDir -Force - } - else { - Write-Host "No PDB files found in $pdbSourceDir" -ForegroundColor Yellow - } - } - else { - Write-Host "PDB source directory not found: $pdbSourceDir (PDBs will only be in .snupkg)" -ForegroundColor Yellow - } + Copy-LibrarySymbols -LibName $LibName -SourceDir $SourceDir Write-Host "" Write-Host "[OK] $LibName packed successfully." -ForegroundColor Green @@ -397,37 +375,39 @@ if ($toPack.Count -gt 0) { if ($Version) { Write-Host "WARNING: -Version is ignored in pack mode (version is detected from produced packages)." -ForegroundColor Yellow } + if (-not $VersionOutputPath) { + throw "Packing local libraries is build-scoped. Run .\build.ps1 -LocalLibraries ." + } - $localRepo = $env:LOCAL_NUGET_REPO + $localRepo = $LocalFeedPath if (-not $localRepo) { - throw "The LOCAL_NUGET_REPO environment variable is not set. Set it to a folder path (e.g. C:\localnugetpackages)." + $localRepo = Get-FieldWorksLocalFeedPath -RepositoryRoot $repoRoot } if (-not (Test-Path $localRepo)) { Write-Host "Creating local NuGet repo folder: $localRepo" -ForegroundColor Yellow New-Item -Path $localRepo -ItemType Directory -Force | Out-Null } - - # Ensure local NuGet source is registered (user-level config) - $sourceList = & dotnet nuget list source 2>&1 - $normalizedRepo = [System.IO.Path]::GetFullPath($localRepo).TrimEnd('\', '/') - $alreadyRegistered = $sourceList | Where-Object { - $_.Trim() -replace '[\\/]$', '' -ieq $normalizedRepo - } - if (-not $alreadyRegistered) { - & dotnet nuget add source $localRepo --name local 2>&1 | Out-Null - Write-Host "Added local NuGet source: $localRepo" -ForegroundColor Yellow - } - Write-Host "" Write-Host "Libraries to pack: $($toPack.Keys -join ', ')" -ForegroundColor Cyan foreach ($lib in $toPack.Keys) { Invoke-PackLibrary -LibName $lib -SourceDir $toPack[$lib] -LocalRepo $localRepo } + $versionOutputDirectory = Split-Path $VersionOutputPath -Parent + if ($versionOutputDirectory -and -not (Test-Path $versionOutputDirectory)) { + New-Item -Path $versionOutputDirectory -ItemType Directory -Force | Out-Null + } + $packedVersions | ConvertTo-Json | Set-Content -LiteralPath $VersionOutputPath ` + -Encoding UTF8 + + # The property file, not the command line, is what a nested restore can see. + Write-FieldWorksLocalLibraryProps ` + -Path (Get-FieldWorksLocalLibraryPropsPath -RepositoryRoot $repoRoot) ` + -Versions $packedVersions -LocalRepository $localRepo Write-Host "" Write-Host "========================================" -ForegroundColor Green - Write-Host "[OK] All libraries packed. Run .\build.ps1 to build." -ForegroundColor Green + Write-Host "[OK] Selected local libraries packed for this build." -ForegroundColor Green Write-Host "========================================" -ForegroundColor Green } elseif ($Library -and $Version) { @@ -450,5 +430,5 @@ elseif ($Library -and $Version) { Write-Host "Run .\build.ps1 to restore and build with the new version." -ForegroundColor Cyan } else { - throw "Nothing to do. Use -Palaso/-Lcm/-Chorus/-Machine/-L10nSharp switches to pack, or -Library and -Version to set a version.`nExamples:`n .\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso`n .\Build\Manage-LocalLibraries.ps1 -Palaso -Chorus`n .\Build\Manage-LocalLibraries.ps1 -Machine -MachinePath C:\Repos\machine`n .\Build\Manage-LocalLibraries.ps1 -Library l10nsharp -Version 10.0.0`n .\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0" + throw "Nothing to do. Use .\build.ps1 -LocalLibraries to pack local libraries, or -Library and -Version to set a version.`nExamples:`n .\build.ps1 -LocalLibraries palaso`n .\build.ps1 -LocalLibraries palaso,chorus`n .\build.ps1 -LocalLibraries machine`n .\Build\Manage-LocalLibraries.ps1 -Library l10nsharp -Version 10.0.0`n .\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0" } diff --git a/Build/Setup-LocalLibraries.ps1 b/Build/Setup-LocalLibraries.ps1 new file mode 100644 index 0000000000..99377a3ebd --- /dev/null +++ b/Build/Setup-LocalLibraries.ps1 @@ -0,0 +1,91 @@ +<# +.SYNOPSIS + Prepares local library checkouts for a FieldWorks build. + +.DESCRIPTION + Takes one or more : pairs, finds each library's git + checkout, and makes the branch available as a worktree. A branch that is + already checked out somewhere is used where it is: git refuses to check one + branch out twice, and another worktree may hold work in progress. Nothing is + ever switched in a checkout that already has a branch, so no work is lost. + + The command reports the resolved paths. It never prompts, so it behaves the + same when run unattended. + +.PARAMETER Library + One or more : pairs, for example lcm:my-fix. + +.PARAMETER Path + Optional explicit checkout path, valid when a single library is named. + +.EXAMPLE + .\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix + Makes the liblcm branch my-fix available and reports its path. + +.EXAMPLE + .\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix,palaso:my-fix + Prepares two libraries in one call. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string[]]$Library, + + [string]$Path +) + +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Force + +$repoRoot = Split-Path $PSScriptRoot -Parent +if ($Path -and $Library.Count -gt 1) { + throw '-Path applies to a single library; name one library or drop -Path.' +} + +$resolved = [ordered]@{} +foreach ($request in $Library) { + $parts = $request -split ':', 2 + if ($parts.Count -ne 2 -or -not $parts[1]) { + throw "Expected : but got '$request' (for example lcm:my-fix)." + } + $name = $parts[0].Trim() + $branch = $parts[1].Trim() + + $repo = Resolve-FieldWorksLibraryRepo -Library $name -RepositoryRoot $repoRoot ` + -ExplicitPath $Path + Write-Host "" + Write-Host "$name" -ForegroundColor Cyan + Write-Host " Checkout: $($repo.Path) (from $($repo.Source))" -ForegroundColor Gray + + $worktree = Resolve-FieldWorksLibraryWorktree -RepositoryPath $repo.Path ` + -Branch $branch -Create + $verb = if ($worktree.Created) { 'created' } else { 'already present' } + Write-Host " Branch: $branch ($verb)" -ForegroundColor Gray + Write-Host " Worktree: $($worktree.Path)" -ForegroundColor Green + + $upstream = Get-FieldWorksLibraryUpstreamState -WorktreePath $worktree.Path + if ($upstream.HasUpstream -and ($upstream.Behind -gt 0 -or $upstream.Ahead -gt 0)) { + # Reported, not merged: a fetch cannot disturb the working tree but a + # merge can, and this worktree may not be the one the caller is editing. + Write-Host (" Upstream: behind {0}, ahead {1} — reconcile it yourself" -f + $upstream.Behind, $upstream.Ahead) -ForegroundColor Yellow + } + + $state = Get-FieldWorksLibrarySourceState -SourceDirectory $worktree.Path + if ($state.IsDirty) { + Write-Host (" Note: {0} uncommitted change(s); builds will repack" -f + $state.DirtyPaths.Count) -ForegroundColor Yellow + } + + $resolved[$name] = $worktree.Path +} + +Write-Host "" +Write-Host "Point this build at the worktrees:" -ForegroundColor Cyan +foreach ($name in $resolved.Keys) { + $envVar = (Get-FieldWorksLocalLibraryConfig)[$name].EnvVar + Write-Host (" `$env:{0} = '{1}'" -f $envVar, $resolved[$name]) +} +Write-Host "" +Write-Host ("Then build with: .\build.ps1 -LocalLibraries {0}" -f + ($resolved.Keys -join ',')) -ForegroundColor Cyan diff --git a/Build/SilVersions.props b/Build/SilVersions.props index 551972abe1..8a5c898621 100644 --- a/Build/SilVersions.props +++ b/Build/SilVersions.props @@ -7,6 +7,7 @@ Imported by: - Directory.Packages.props (managed NuGet CPM) - Build/mkall.targets (native C++ build) + - Build/PackageRestore.targets (restore, including nested ones) Update a version here to update ALL consumers automatically. ============================================================= @@ -24,4 +25,12 @@ 70.1.152 60.0.56 + + + diff --git a/Docs/architecture/dependencies.md b/Docs/architecture/dependencies.md index 140ae06601..58fed1ccae 100644 --- a/Docs/architecture/dependencies.md +++ b/Docs/architecture/dependencies.md @@ -34,14 +34,16 @@ By default, dependencies are downloaded as NuGet packages during the build. The ## Building and Debugging Dependencies Locally -If you need to debug into or modify a dependency library, use the `Build/Manage-LocalLibraries.ps1` script. It packs a local checkout into a local NuGet feed, detects the produced version, and updates `SilVersions.props` to match. +If you need to debug into or modify a dependency library, select it with +`build.ps1 -LocalLibraries`. The build packs the local checkout into a local NuGet +feed inside this working tree and uses its derived version for that invocation +without changing `SilVersions.props`. Quick start: ```powershell -$env:LOCAL_NUGET_REPO = "C:\localnugetpackages" -.\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso -.\build.ps1 +$env:LIBPALASO_PATH = "C:\Repos\libpalaso" +.\build.ps1 -LocalLibraries palaso ``` For the full workflow (setup, pack, build, debug, revert), see **[Local Library Debugging](local-library-debugging.md)**. diff --git a/Docs/architecture/local-library-debugging.md b/Docs/architecture/local-library-debugging.md index 02b2fba90d..3621d408f6 100644 --- a/Docs/architecture/local-library-debugging.md +++ b/Docs/architecture/local-library-debugging.md @@ -1,149 +1,140 @@ # Local Library Debugging -This document describes how to debug locally-modified versions of **liblcm**, **libpalaso**, **chorus**, or **machine** (SIL.Machine) in FieldWorks using a local NuGet feed. +Use `build.ps1 -LocalLibraries` to rebuild locally modified SIL libraries and +use them for one FieldWorks build. A later build that does not select a library +automatically removes its local packages and restores the published version. -## Overview +## One-time setup -The workflow uses a single PowerShell script (`Build/Manage-LocalLibraries.ps1`) that: +Packed packages go to `.localfeed` inside this working tree, so no feed setup is +needed. The feed is per working tree on purpose: a machine-wide folder lets one +working tree's build delete packages another working tree just produced. +`LOCAL_NUGET_REPO` still overrides the location when it is set, but sharing one +folder across working trees reintroduces that problem. -1. Adds a local NuGet source to `nuget.config` (pointing to your `LOCAL_NUGET_REPO` folder). -2. Runs `dotnet pack` in Debug configuration with symbols, letting the library use its own version. -3. Detects the version from the produced packages. -4. Updates `SilVersions.props` so FieldWorks resolves that exact version. -5. Places `.nupkg` / `.snupkg` in your local NuGet feed folder. -6. Copies PDB files to `Output/Debug/` and `Downloads/` for debugger access. -7. Clears stale cached packages so the next restore picks up the local build. +## Prepare a library branch -This approach works identically for all three libraries. - -## Setup (one-time) - -### 1. Create a local NuGet folder - -Pick any folder, for example: - -``` -C:\localnugetpackages -``` - -### 2. Set the `LOCAL_NUGET_REPO` environment variable +`Setup-LocalLibraries.ps1` finds a library's checkout and makes a branch +available as a worktree: ```powershell -# Current session -$env:LOCAL_NUGET_REPO = "C:\localnugetpackages" - -# Persistent (user-level) -[System.Environment]::SetEnvironmentVariable("LOCAL_NUGET_REPO", "C:\localnugetpackages", "User") +.\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix +.\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix,palaso:my-fix ``` -The script automatically registers this folder as a NuGet source in your user-level NuGet config when you pack. The repo's `nuget.config` is not modified. - -### 3. Clone the library you need - -```powershell -git clone https://github.com/sillsdev/liblcm.git -git clone https://github.com/sillsdev/libpalaso.git -git clone https://github.com/sillsdev/chorus.git -git clone https://github.com/sillsdev/machine.git -``` - -## Pack a local library - -```powershell -# Single library — explicit path -.\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso +It looks for the checkout in the path you pass, then beside the FieldWorks +checkout (`../liblcm`), then in the library's path variable. If the branch is +already checked out somewhere it uses that worktree as it stands, because git +refuses to check one branch out twice and because that worktree may hold work in +progress. Otherwise it creates one under the library's `.tmp/worktrees/`. -# Multiple libraries (libpalaso is always packed first) -.\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso -Chorus -ChorusPath C:\Repos\chorus -``` +The command fetches but never merges, and never switches a branch in a checkout +that already has one, so it cannot disturb work you have not committed. It +reports what it found and never waits for input, so it behaves the same when run +unattended. It prints the path variable to set for the worktree it resolved. -Or set environment variables so you can omit the paths: +Set the path variable for each local checkout you use: ```powershell $env:LIBPALASO_PATH = "C:\Repos\libpalaso" $env:LIBLCM_PATH = "C:\Repos\liblcm" $env:LIBCHORUS_PATH = "C:\Repos\chorus" $env:SILMACHINE_PATH = "C:\Repos\machine" - -# Switches still required — env vars only provide the path -.\Build\Manage-LocalLibraries.ps1 -Palaso -Chorus +$env:L10NSHARP_PATH = "C:\Repos\L10NSharp" ``` -The script: -- Lets the library build with its own version (no version override). -- Detects the produced version and updates `Build/SilVersions.props` to match. -- Produces `.snupkg` symbol packages (same format as production). -- Copies PDB files to `Output/Debug/` and `Downloads/` for the debugger. -- Clears stale packages from the `packages/` cache. +## Build with local libraries -## Build FieldWorks +Name every local library that this invocation should use: ```powershell -.\build.ps1 -``` +# Rebuild and use Machine locally for this build. +.\build.ps1 -LocalLibraries machine -The build will print a yellow message listing any local packages detected in `LOCAL_NUGET_REPO`. NuGet restore will use your local packages because `SilVersions.props` was updated to request the exact version produced by the library. +# Rebuild and use Palaso and Chorus locally for this build. +.\build.ps1 -LocalLibraries palaso,chorus +``` -## Debug +Every selected library is repacked from its configured checkout. The packed +versions are passed to restore and MSBuild without modifying +`Build/SilVersions.props`. The local feed is a restore source only for that +invocation. -1. Open FieldWorks in Visual Studio. -2. PDB files are already in `Output/Debug/` — the debugger will find them automatically. -3. If breakpoints show "No symbols loaded", disable **Debug > Options > Enable Just My Code**. -4. You can also open the library solution side-by-side and use **Debug > Attach to Process**. +Each pack is versioned from the checkout it came from, as +`-.` — for example `3.9.2-docs-hc-llm-guide.9358825`. This +matters because NuGet resolves an already-extracted `(id, version)` in +`packages/` before it consults a folder feed: if a local pack reused the +published version string, restore would silently serve whichever copy landed +first. Deriving the version from the branch and commit makes the two +distinguishable, and makes repacking the same commit a correct cache hit. -## Iterating +Each selected library is built before it is packed, because a package may +include output from a target framework its own project does not build. -After each change to the library: +The core comes from the library's own GitVersion where it has one, so a version +bump in the library is reflected in the local package. A library without +GitVersion, such as SIL.Machine, falls back to the version FieldWorks consumes. -1. Re-run `Manage-LocalLibraries.ps1` (~30-60 seconds). -2. Re-run `.\build.ps1`. +Because a clean commit identifies its contents, a second build from the same +commit reuses the package already in the feed instead of repacking it. -## Setting a specific version +A checkout with uncommitted changes has no stable identity, so it is packed as +`-.dirty` and repacked on every build. The build lists the paths +responsible, because committing or removing them is what restores the faster +path. Untracked files count, which means anything `.gitignore` does not cover: +a stray note beside the source is enough to keep repacking. -Use `-Version` to set any library to a specific version in `SilVersions.props` without packing: +An ordinary build selects no local libraries: ```powershell -# Revert libpalaso to an upstream version -.\Build\Manage-LocalLibraries.ps1 -Library libpalaso -Version 17.0.0 - -# Set liblcm to a specific pre-release version -.\Build\Manage-LocalLibraries.ps1 -Library liblcm -Version 11.0.0-beta0159 +.\build.ps1 ``` -This updates `SilVersions.props` and clears stale cached packages. Run `.\build.ps1` afterward to restore and build with the new version. +Before restore, every build removes managed packages from the local feed and +removes managed cache versions whose NuGet metadata identifies a filesystem +source. Published packages restored from an HTTP source stay cached, so an +ordinary build does not redownload them every time. Package sources inherited +from user-level configuration are left alone, because a local build adds its own +feed for that build only. -## Reverting to upstream packages +The selected versions and that feed are written to a generated +`Build/LocalLibraries.props`, which `Build/SilVersions.props` imports. They are +written to a file rather than passed on the command line because a restore +launched through `Exec` is a new MSBuild process and does not inherit global +properties, so a version passed that way would not reach it. -Use `-Version` to set the library back to its upstream version: +## Debug and iterate -```powershell -.\Build\Manage-LocalLibraries.ps1 -Library libpalaso -Version 17.0.0 -``` +The local build copies PDBs to `Output/Debug/` and `Downloads/`. If Visual +Studio reports that symbols are not loaded, disable **Debug > Options > Enable +Just My Code**. -Or revert all libraries at once: +After each local-library change, rerun `build.ps1` with the same +`-LocalLibraries` selection. Omit a library whenever FieldWorks should return to +its published package. -```powershell -git checkout Build/SilVersions.props -Remove-Item -Recurse packages/sil.* -.\build.ps1 -``` +## Set an explicit published version -To also remove the user-level local source: +The lower-level script still supports changing `SilVersions.props` deliberately: ```powershell -dotnet nuget remove source local +.\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0 ``` +Local packing through `Manage-LocalLibraries.ps1` is build-internal. Run +`build.ps1 -LocalLibraries` instead. + ## Supported libraries -| Library | Switch | Path parameter | Version property | Env var fallback | -|---------|--------|---------------|------------------|-----------------| -| liblcm | `-Lcm` | `-LcmPath` | `SilLcmVersion` | `LIBLCM_PATH` | -| libpalaso | `-Palaso` | `-PalasoPath` | `SilLibPalasoVersion` | `LIBPALASO_PATH` | -| chorus | `-Chorus` | `-ChorusPath` | `SilChorusVersion` | `LIBCHORUS_PATH` | -| machine | `-Machine` | `-MachinePath` | `SilMachineVersion` | `SILMACHINE_PATH` | +| Library | Selection | Version property | Checkout environment variable | +|---------|-----------|------------------|-------------------------------| +| libpalaso | `palaso` | `SilLibPalasoVersion` | `LIBPALASO_PATH` | +| L10NSharp | `l10nsharp` | `L10NSharpVersion` | `L10NSHARP_PATH` | +| liblcm | `lcm` | `SilLcmVersion` | `LIBLCM_PATH` | +| chorus | `chorus` | `SilChorusVersion` | `LIBCHORUS_PATH` | +| Machine | `machine` | `SilMachineVersion` | `SILMACHINE_PATH` | -## See Also +## See also -- [Dependencies](dependencies.md) — overview of external dependencies -- [Build Instructions](../../.github/instructions/build.instructions.md) — building FieldWorks +- [Dependencies](dependencies.md) +- [Build Instructions](../../.github/instructions/build.instructions.md) diff --git a/build.ps1 b/build.ps1 index 3ff6eaa985..5b5783802f 100644 --- a/build.ps1 +++ b/build.ps1 @@ -114,6 +114,10 @@ Path to the local liblcm repository. Defaults to ../liblcm relative to the FieldWorks repo root. Only used when -UseLocalLcm is specified. +.PARAMETER LocalLibraries + Local SIL libraries to rebuild and use for this invocation. Supported values are + palaso, lcm, chorus, machine, and l10nsharp. Omitted libraries are cleaned before restore. + .PARAMETER StartedBy Optional actor label written to the worktree lock metadata (for example: user or agent). Defaults to the FW_BUILD_STARTED_BY environment variable when set, otherwise 'unknown'. @@ -153,6 +157,10 @@ .\build.ps1 -UseLocalLcm Builds FieldWorks, then builds liblcm from ../liblcm and copies DLLs into Output. +.EXAMPLE + .\build.ps1 -LocalLibraries machine + Rebuilds Machine from SILMACHINE_PATH and uses it only for this build. + .NOTES FieldWorks is x64-only. The x86 platform is no longer supported. #> @@ -191,6 +199,8 @@ param( [switch]$EnableTracing, [switch]$UseLocalLcm, [string]$LocalLcmPath, + [ValidateSet('palaso', 'lcm', 'chorus', 'machine', 'l10nsharp')] + [string[]]$LocalLibraries = @(), [ValidateSet('user', 'agent', 'unknown')] [string]$StartedBy = 'unknown', [switch]$SkipWorktreeLock, @@ -575,6 +585,58 @@ try { & $staleDllScript -OutputDir $outputDir -RepoRoot $PSScriptRoot -Verbose:$VerbosePreference } + $localLibrariesModule = Join-Path $PSScriptRoot 'Build/LocalLibraries.psm1' + Import-Module $localLibrariesModule -Force + $packagesDir = Join-Path $PSScriptRoot 'packages' + $localFeed = Get-FieldWorksLocalFeedPath -RepositoryRoot $PSScriptRoot + $localPropsPath = Get-FieldWorksLocalLibraryPropsPath -RepositoryRoot $PSScriptRoot + + # Only when this build will restore. Removing packages that nothing is + # going to put back would leave the tree unbuildable. + if (-not $SkipRestore) { + Clear-FieldWorksLocalLibraries -PackagesDirectory $packagesDir ` + -LocalRepository $localFeed + + # An earlier selection goes first, so a build naming no library + # resolves published versions even in a restore it does not launch. + Remove-FieldWorksLocalLibraryProps -Path $localPropsPath + } + + $localVersionProperties = [ordered]@{} + $localRestoreSourceArg = $null + if ($LocalLibraries.Count -gt 0) { + if ($SkipRestore) { + throw '-LocalLibraries cannot be combined with -SkipRestore.' + } + if ($UseLocalLcm -and $LocalLibraries -contains 'lcm') { + throw 'Choose either -LocalLibraries lcm or -UseLocalLcm, not both.' + } + $versionOutputPath = Join-Path ([System.IO.Path]::GetTempPath()) ` + ("FieldWorksLocalVersions_{0}.json" -f [System.Guid]::NewGuid().ToString('N')) + $managerArgs = @{ VersionOutputPath = $versionOutputPath + LocalFeedPath = $localFeed } + foreach ($localLibrary in $LocalLibraries) { + $managerArgs[$localLibrary] = $true + } + try { + # No $LASTEXITCODE check: that script reports failure by throwing, + # so the variable would hold the last native command's result. + & (Join-Path $PSScriptRoot 'Build/Manage-LocalLibraries.ps1') @managerArgs + $versionOutput = Get-Content -LiteralPath $versionOutputPath -Raw | + ConvertFrom-Json + foreach ($property in $versionOutput.PSObject.Properties) { + $localVersionProperties[$property.Name] = [string]$property.Value + } + } + finally { + if (Test-Path -LiteralPath $versionOutputPath) { + Remove-Item -LiteralPath $versionOutputPath -Force + } + } + $localRestoreSourceArg = + "/p:RestoreAdditionalProjectSources=$localFeed" + } + # ============================================================================= # Build Configuration # ============================================================================= @@ -616,6 +678,9 @@ try { # Properties $finalMsBuildArgs += "/p:Configuration=$Configuration" $finalMsBuildArgs += "/p:Platform=$Platform" + foreach ($propertyName in $localVersionProperties.Keys) { + $finalMsBuildArgs += "/p:$propertyName=$($localVersionProperties[$propertyName])" + } if ($SkipNative) { $finalMsBuildArgs += "/p:SkipNative=true" } @@ -657,18 +722,12 @@ try { Write-Host "Including optional FieldWorks executables" -ForegroundColor Yellow } - # Report local library packages when LOCAL_NUGET_REPO is configured - if ($env:LOCAL_NUGET_REPO -and (Test-Path $env:LOCAL_NUGET_REPO)) { - $localPkgs = Get-ChildItem -Path $env:LOCAL_NUGET_REPO -Filter "SIL.*.nupkg" -File -ErrorAction SilentlyContinue - if ($localPkgs.Count -gt 0) { - Write-Host "" - Write-Host "Local library packages detected in $($env:LOCAL_NUGET_REPO):" -ForegroundColor Yellow - foreach ($pkg in $localPkgs) { - Write-Host " $($pkg.Name)" -ForegroundColor Yellow - } - Write-Host "These will shadow upstream NuGet packages during restore." -ForegroundColor Yellow - Write-Host "" - } + if ($LocalLibraries.Count -gt 0) { + # Say how to undo it now, while the person who chose it is here. + Write-Host "To go back to published packages, build without -LocalLibraries." ` + -ForegroundColor Cyan + Write-Host "Using local libraries: $($LocalLibraries -join ', ')" ` + -ForegroundColor Yellow } # Bootstrap: Build FwBuildTasks first (required by SetupInclude.targets) @@ -705,7 +764,21 @@ try { if (-not (Test-Path $packagesDir)) { New-Item -Path $packagesDir -ItemType Directory -Force | Out-Null } - & dotnet restore "$PSScriptRoot\FieldWorks.sln" /p:NoWarn=NU1903 /p:DisableWarnForInvalidRestoreProjects=true "/p:Configuration=$Configuration" "/p:Platform=$Platform" --verbosity quiet + $restoreArgs = @( + "$PSScriptRoot\FieldWorks.sln", + '/p:NoWarn=NU1903', + '/p:DisableWarnForInvalidRestoreProjects=true', + "/p:Configuration=$Configuration", + "/p:Platform=$Platform", + '--verbosity', 'quiet' + ) + foreach ($propertyName in $localVersionProperties.Keys) { + $restoreArgs += "/p:$propertyName=$($localVersionProperties[$propertyName])" + } + if ($localRestoreSourceArg) { + $restoreArgs += $localRestoreSourceArg + } + & dotnet restore @restoreArgs if ($LASTEXITCODE -ne 0) { throw "NuGet package restore failed for FieldWorks.sln" } diff --git a/nuget.config b/nuget.config index 3d86798cf0..ce6b66f7b2 100644 --- a/nuget.config +++ b/nuget.config @@ -30,9 +30,9 @@ diff --git a/test.ps1 b/test.ps1 index 10f99419ef..54a8745acb 100644 --- a/test.ps1 +++ b/test.ps1 @@ -125,6 +125,15 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# Only on a full run: a caller who named a project or filter asked for that. +if (-not $TestProject -and -not $TestFilter) { + $localLibrariesTestPath = Join-Path $PSScriptRoot "Build/LocalLibraries.Tests.ps1" + & $localLibrariesTestPath + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } +} + if (-not $PSBoundParameters.ContainsKey('StartedBy') -and -not [string]::IsNullOrWhiteSpace($env:FW_BUILD_STARTED_BY)) { $startedByFromEnv = $env:FW_BUILD_STARTED_BY.ToLowerInvariant() if ($startedByFromEnv -in @('user', 'agent', 'unknown')) {