diff --git a/.gitignore b/.gitignore index a4c7c89ff4..1ebb84e0b8 100644 --- a/.gitignore +++ b/.gitignore @@ -223,3 +223,8 @@ DataTreeTimingBaselines.json Docs/migration/working/ Build/Agent/comment-hygiene-report.json .review/ + +# Working-tree-local NuGet feed for -LocalLibraries packs +.localfeed/ +# Generated by build.ps1 -LocalLibraries; selects local library versions +Build/LocalLibraries.props diff --git a/Build/Agent/powershell-compat.ps1 b/Build/Agent/powershell-compat.ps1 index dac023085e..159aa20cfc 100644 --- a/Build/Agent/powershell-compat.ps1 +++ b/Build/Agent/powershell-compat.ps1 @@ -48,11 +48,14 @@ $ErrorActionPreference = 'Stop' Import-Module (Join-Path $PSScriptRoot 'CommentHygiene.psm1') -Force $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path -# Every PowerShell file under Build/Agent, so a new script is covered the day it -# lands rather than when someone remembers to name it here. -$targetFiles = @(Get-ChildItem -LiteralPath $PSScriptRoot -Recurse -File | +# Every PowerShell file under Build, not just Build/Agent: build.ps1 loads +# modules from there under Windows PowerShell 5.1 in CI. +$scanRoots = @($PSScriptRoot, (Join-Path $repoRoot 'Build')) +$targetFiles = @($scanRoots | ForEach-Object { + Get-ChildItem -LiteralPath $_ -Recurse -File -ErrorAction SilentlyContinue + } | Where-Object { $_.Extension -eq '.ps1' -or $_.Extension -eq '.psm1' } | - ForEach-Object { $_.FullName } | Sort-Object) + ForEach-Object { $_.FullName } | Sort-Object -Unique) $violations = New-Object System.Collections.ArrayList diff --git a/Build/LocalLibraries.Tests.ps1 b/Build/LocalLibraries.Tests.ps1 new file mode 100644 index 0000000000..72e2b6bed5 --- /dev/null +++ b/Build/LocalLibraries.Tests.ps1 @@ -0,0 +1,343 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$failures = New-Object System.Collections.ArrayList +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ( + 'FieldWorksLocalLibrariesTests_' + [System.Guid]::NewGuid().ToString('N')) + +function Assert-True { + param([bool]$Condition, [string]$Message) + if (-not $Condition) { + [void]$script:failures.Add("FAIL: $Message") + } +} + +function Write-PackageMetadata { + param([string]$VersionDirectory, [string]$Source) + New-Item -ItemType Directory -Path $VersionDirectory -Force | Out-Null + @{ version = 2; contentHash = 'test'; source = $Source } | + ConvertTo-Json | Set-Content -LiteralPath ( + Join-Path $VersionDirectory '.nupkg.metadata') -Encoding UTF8 +} + +try { + $packagesDirectory = Join-Path $tempRoot 'packages' + $localRepository = Join-Path $tempRoot 'feed' + New-Item -ItemType Directory -Path $localRepository -Force | Out-Null + + $localMachine = Join-Path $packagesDirectory 'sil.machine\3.9.2' + $publishedMachine = Join-Path $packagesDirectory 'sil.machine\3.9.3' + $unrelatedPackage = Join-Path $packagesDirectory 'example.package\1.0.0' + Write-PackageMetadata -VersionDirectory $localMachine -Source $localRepository + Write-PackageMetadata -VersionDirectory $publishedMachine ` + -Source 'https://api.nuget.org/v3/index.json' + Write-PackageMetadata -VersionDirectory $unrelatedPackage -Source $localRepository + + Set-Content -LiteralPath (Join-Path $localRepository 'SIL.Machine.3.9.2.nupkg') ` + -Value 'local package' + Set-Content -LiteralPath ( + Join-Path $localRepository 'SIL.Machine.Morphology.HermitCrab.3.9.2.snupkg') ` + -Value 'local symbols' + Set-Content -LiteralPath (Join-Path $localRepository 'Example.Package.1.0.0.nupkg') ` + -Value 'unrelated package' + $managedFeedPackages = @( + 'SIL.Core.18.0.0.nupkg', + 'SIL.LCModel.11.0.0.nupkg', + 'SIL.Chorus.LibChorus.6.0.0.nupkg', + 'L10NSharp.10.0.0.nupkg' + ) + foreach ($packageName in $managedFeedPackages) { + Set-Content -LiteralPath (Join-Path $localRepository $packageName) ` + -Value 'managed package' + } + + Import-Module (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Force + $config = Get-FieldWorksLocalLibraryConfig + Assert-True ($config.Keys.Count -eq 5) 'The catalogue should contain five libraries.' + foreach ($library in @('palaso', 'lcm', 'chorus', 'machine', 'l10nsharp')) { + Assert-True $config.Contains($library) "The catalogue should contain $library." + } + + Clear-FieldWorksLocalLibraries -PackagesDirectory $packagesDirectory ` + -LocalRepository $localRepository + + Assert-True (-not (Test-Path $localMachine)) ` + 'Cleanup should remove cache entries restored from a filesystem source.' + Assert-True (Test-Path $publishedMachine) ` + 'Cleanup should preserve cache entries restored from an HTTP source.' + Assert-True (Test-Path $unrelatedPackage) ` + 'Cleanup should preserve packages outside the managed library catalogue.' + Assert-True (-not (Test-Path ( + Join-Path $localRepository 'SIL.Machine.3.9.2.nupkg'))) ` + 'Cleanup should remove managed packages from the local feed.' + Assert-True (-not (Test-Path ( + Join-Path $localRepository 'SIL.Machine.Morphology.HermitCrab.3.9.2.snupkg'))) ` + 'Cleanup should remove managed symbol packages from the local feed.' + Assert-True (Test-Path (Join-Path $localRepository 'Example.Package.1.0.0.nupkg')) ` + 'Cleanup should preserve unrelated packages in the local feed.' + foreach ($packageName in $managedFeedPackages) { + Assert-True (-not (Test-Path (Join-Path $localRepository $packageName))) ` + "Cleanup should remove $packageName." + } + + Clear-FieldWorksLibraryPackageCache -PackagesDirectory $packagesDirectory ` + -Libraries @('machine') + Assert-True (-not (Test-Path $publishedMachine)) ` + 'Selected packing should evict a published cache entry with the same version.' + Assert-True (Test-Path $unrelatedPackage) ` + 'Selected packing should preserve cache entries outside its package family.' + + $managerText = Get-Content -LiteralPath ( + Join-Path $PSScriptRoot 'Manage-LocalLibraries.ps1') -Raw + Assert-True ($managerText -match '\$VersionOutputPath') ` + 'Manage-LocalLibraries should accept a packed-version output path.' + Assert-True ($managerText -match 'Import-Module.+LocalLibraries\.psm1') ` + 'Manage-LocalLibraries should import the shared library catalogue.' + Assert-True ($managerText -match 'Clear-FieldWorksLocalLibraries') ` + 'Manage-LocalLibraries should use the shared cache cleanup.' + Assert-True ($managerText -match 'Clear-FieldWorksLibraryPackageCache') ` + 'Pack mode should evict selected package families before local restore.' + Assert-True ($managerText -match 'Packing local libraries is build-scoped') ` + 'Direct pack mode should direct callers to build.ps1.' + Assert-True ($managerText -notmatch 'dotnet nuget add source') ` + 'Manage-LocalLibraries should not persist a user-level NuGet source.' + Assert-True ($managerText -match 'RestoreAdditionalProjectSources=\$LocalRepo') ` + 'Local pack restores should receive the local feed for dependent libraries.' + + $buildText = Get-Content -LiteralPath (Join-Path $PSScriptRoot '..\build.ps1') -Raw + Assert-True ($buildText -match '\[string\[\]\]\$LocalLibraries') ` + 'build.ps1 should accept a LocalLibraries array.' + Assert-True ($buildText -match 'Clear-FieldWorksLocalLibraries') ` + 'build.ps1 should clean unselected local libraries before restore.' + Assert-True ($buildText -match 'VersionOutputPath') ` + 'build.ps1 should consume non-persistent packed version output.' + Assert-True (($buildText -match 'Get-FieldWorksLocalFeedPath') -and ` + ($buildText -match 'RestoreAdditionalProjectSources')) ` + 'build.ps1 should add the resolved local feed to configured restore sources.' + Assert-True ($buildText -notmatch '\$env:LOCAL_NUGET_REPO') ` + 'build.ps1 should resolve the feed through the module, not the env var.' + + # --- Feed location ----------------------------------------------------- + # A machine-wide feed lets one working tree delete another's fresh packages. + $savedFeed = $env:LOCAL_NUGET_REPO + try { + $env:LOCAL_NUGET_REPO = $null + Assert-True ((Get-FieldWorksLocalFeedPath -RepositoryRoot 'C:\wt') -eq ` + 'C:\wt\.localfeed') ` + 'The feed should default inside the working tree.' + $env:LOCAL_NUGET_REPO = 'C:/legacy/feed' + Assert-True ((Get-FieldWorksLocalFeedPath -RepositoryRoot 'C:\wt') -eq ` + 'C:/legacy/feed') ` + 'An explicit LOCAL_NUGET_REPO should still win.' + } + finally { + $env:LOCAL_NUGET_REPO = $savedFeed + } + + # --- Version labels ---------------------------------------------------- + # SemVer pre-release identifiers allow only alphanumerics and hyphens. + Assert-True ((ConvertTo-FieldWorksVersionLabel -BranchName 'feature/x_y') -eq ` + 'feature-x-y') 'Branch separators should become hyphens.' + Assert-True ((ConvertTo-FieldWorksVersionLabel -BranchName 'LT-1/2') -eq ` + 'lt-1-2') 'Labels should be lowercased.' + Assert-True ((ConvertTo-FieldWorksVersionLabel ` + -BranchName ('a' * 60)).Length -le 24) 'Long branches should truncate.' + Assert-True ((ConvertTo-FieldWorksVersionLabel -BranchName '///') -eq ` + 'detached') 'A branch with no usable characters should be named.' + + # --- Pack versions --- + # A clean tree is identified by its commit, so its cache entry may be + # reused; a dirty tree has no stable identity. + $cleanState = [pscustomobject]@{ + Branch = 'feature/x'; Label = 'feature-x'; ShortSha = 'abc1234'; IsDirty = $false } + $dirtyState = [pscustomobject]@{ + Branch = 'feature/x'; Label = 'feature-x'; ShortSha = 'abc1234'; IsDirty = $true } + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '3.9.2' ` + -SourceState $cleanState) -eq '3.9.2-feature-x.abc1234') ` + 'A clean pack should be identified by branch and commit.' + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '11.0.0-beta0178' ` + -SourceState $cleanState) -eq '11.0.0-feature-x.abc1234') ` + 'A published pre-release label should be replaced, not appended to.' + # The core comes from the library's own GitVersion where it has one, so a + # version bump in the library shows up in the local package. + $config = Get-FieldWorksLocalLibraryConfig + Assert-True ($config['lcm'].VersionProject -eq 'src/SIL.LCModel/SIL.LCModel.csproj') ` + 'GitVersion libraries should name a project to read their version from.' + Assert-True (-not $config['machine'].Contains('VersionProject')) ` + 'A library without GitVersion should fall back to the consumed version.' + Assert-True ((Get-FieldWorksLibraryCoreVersion -SourceDirectory 'C: +ope' ` + -LibraryEntry $config['machine'] -FallbackVersion '3.9.2') -eq '3.9.2') ` + 'The fallback should be the consumed version core.' + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '3.9.2' ` + -SourceState $dirtyState) -eq '3.9.2-feature-x.dirty') ` + 'A dirty pack should be marked dirty rather than pinned to a commit.' + Assert-True ((Get-FieldWorksLocalPackVersion -CoreVersion '3.9.2' ` + -SourceState $cleanState) -ne '3.9.2') ` + 'A local pack must never reuse the published version string.' + + Assert-True ($managerText -match '-p:Version=\$packVersion') ` + 'Pack should stamp the derived version instead of sniffing filenames.' + Assert-True ($managerText -notmatch 'function Get-PackageVersion') ` + 'Pack should not parse versions out of filenames any more.' + # An earlier local pack is filesystem-sourced, so the pre-pack cleanup + # already removes it; clearing again would evict published packages too. + Assert-True (([regex]::Matches($managerText, + 'Clear-FieldWorksLibraryPackageCache')).Count -eq 1) ` + 'Only SetVersion mode should clear every cached version of a library.' + + # --- Pack reuse --- + # A clean commit identifies its contents, so an existing package for that + # version is the package this pack would produce. + $reuseFeed = Join-Path $tempRoot 'reusefeed' + New-Item -ItemType Directory -Force $reuseFeed | Out-Null + Set-Content -LiteralPath (Join-Path $reuseFeed 'SIL.Machine.3.9.2-b.abc1234.nupkg') ` + -Value 'x' + Assert-True (Test-FieldWorksPackIsCurrent -LocalRepository $reuseFeed ` + -PackVersion '3.9.2-b.abc1234' -SourceState $cleanState) ` + 'An existing package for a clean commit should be reused.' + Assert-True (-not (Test-FieldWorksPackIsCurrent -LocalRepository $reuseFeed ` + -PackVersion '3.9.2-b.abc1234' -SourceState $dirtyState)) ` + 'A dirty tree must never reuse an existing package.' + Assert-True (-not (Test-FieldWorksPackIsCurrent -LocalRepository $reuseFeed ` + -PackVersion '3.9.2-b.9999999' -SourceState $cleanState)) ` + 'A different commit should not be satisfied by another version.' + Assert-True ($managerText -match 'Reusing the packed') ` + 'Pack should report when it reuses an existing package.' + Assert-True ($managerText -match 'Uncommitted changes force a repack') ` + 'Pack should name the paths that force a repack.' + + # --- Worktree setup --- + # Branch names may contain path separators; the directory keeps the full + # name so a developer recognises which worktree is which. + Assert-True ((ConvertTo-FieldWorksWorktreeName -BranchName 'feature/x') -eq ` + 'feature-x') 'A branch name should become one directory name.' + Assert-True ((ConvertTo-FieldWorksWorktreeName ` + -BranchName 'LT-22728-keep-full-name') -eq 'LT-22728-keep-full-name') ` + 'A worktree name should not be truncated or lowercased.' + Assert-True ($config['lcm'].RepoDirectory -eq 'liblcm') ` + 'Each library should name its sibling checkout directory.' + + # --- One catalogue, three declarations --- + # The library names appear in the catalogue and in two ValidateSets, so + # each must agree with the catalogue. + $catalogue = @($config.Keys | Sort-Object) + foreach ($source in @(@{ n = 'build.ps1'; t = $buildText }, + @{ n = 'Manage-LocalLibraries.ps1'; t = $managerText })) { + $declared = [regex]::Matches($source.t, + "\[ValidateSet\((?'(?:palaso|lcm|chorus|machine|l10nsharp)'[^)]*)\)\]") + Assert-True ($declared.Count -ge 1) ` + "$($source.n) should declare the library names." + if ($declared.Count -ge 1) { + $names = @([regex]::Matches($declared[0].Groups['set'].Value, "'([^']+)'") | + ForEach-Object { $_.Groups[1].Value } | Sort-Object) + Assert-True (($names -join ',') -eq ($catalogue -join ',')) ` + "$($source.n) ValidateSet should match the catalogue exactly." + } + } + + # --- Odd cache entries --- + # Version 1 metadata records no source, and reading a missing property + # under Set-StrictMode is terminating. + $oddRoot = Join-Path $tempRoot 'odd' + $v1 = Join-Path $oddRoot 'packages/sil.lcmodel/1.0.0' + New-Item -ItemType Directory -Path $v1 -Force | Out-Null + '{ "version": 1, "contentHash": "abc" }' | + Set-Content -LiteralPath (Join-Path $v1 '.nupkg.metadata') -Encoding UTF8 + $partial = Join-Path $oddRoot 'packages/sil.lcmodel/2.0.0' + New-Item -ItemType Directory -Path $partial -Force | Out-Null + Clear-FieldWorksLocalLibraries -PackagesDirectory (Join-Path $oddRoot 'packages') ` + -LocalRepository (Join-Path $oddRoot 'feed') -Libraries @('lcm') + Assert-True (Test-Path -LiteralPath $v1) ` + 'Version 1 metadata should be preserved, not crash the build.' + Assert-True (-not (Test-Path -LiteralPath $partial)) ` + 'A version directory with no metadata is a partial extraction and should go.' + + # --- Reaching a restore this build does not launch --- + # A restore run through Exec is a new process, so an override has to be on + # disk where every process reads it, not on a command line. + $silVersions = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'SilVersions.props') -Raw + Assert-True ($silVersions -match 'LocalLibraries\.props') ` + 'SilVersions.props should import the generated local library overrides.' + Assert-True ($silVersions.IndexOf('LocalLibraries.props') -gt ` + $silVersions.IndexOf('SilLcmVersion')) ` + 'The override must be imported after the defaults so that it wins.' + $propsPath = Join-Path $tempRoot 'gen/Build/LocalLibraries.props' + Write-FieldWorksLocalLibraryProps -Path $propsPath ` + -Versions ([ordered]@{ SilLcmVersion = '9.9.9-x.abc1234' }) ` + -LocalRepository 'C:/feed' + [xml]$generated = Get-Content -LiteralPath $propsPath + Assert-True ($generated.Project.PropertyGroup.SilLcmVersion -eq '9.9.9-x.abc1234') ` + 'The generated overrides should carry the packed version.' + Assert-True ($generated.Project.PropertyGroup.RestoreAdditionalProjectSources ` + -match 'C:/feed') 'The generated overrides should carry the local feed.' + Remove-FieldWorksLocalLibraryProps -Path $propsPath + Assert-True (-not (Test-Path -LiteralPath $propsPath)) ` + 'A build selecting no library should remove the overrides.' + Assert-True ($buildText -match 'Remove-FieldWorksLocalLibraryProps') ` + 'Every build should clear a previous selection before restoring.' + + # --- Symbol directories --- + # A directory that does not exist copies nothing and says nothing, so the + # configured paths have to match where each library actually writes. + foreach ($name in $config.Keys) { + foreach ($relative in @($config[$name].PdbRelativeDir)) { + Assert-True ($relative -notmatch 'net462$' -or + $name -in @('palaso', 'lcm', 'chorus')) ` + "$name should not claim a net462 symbol directory." + } + } + Assert-True (@($config['machine'].PdbRelativeDir).Count -eq 2) ` + 'A library with per-project output needs one symbol directory per project.' + Assert-True ($config['l10nsharp'].PdbRelativeDir -eq 'output/Debug/net48') ` + 'L10NSharp symbols should come from a framework it actually builds.' + Assert-True ($managerText -match 'No PDB files found for') ` + 'A missing symbol directory should be reported, not passed over.' + + $testText = Get-Content -LiteralPath (Join-Path (Split-Path $PSScriptRoot -Parent) 'test.ps1') -Raw + Assert-True ($testText -match 'if \(-not \$TestProject -and -not \$TestFilter\)') ` + 'A targeted test run should not also run the local library tests.' + + $setupText = Get-Content -LiteralPath ` + (Join-Path $PSScriptRoot 'Setup-LocalLibraries.ps1') -Raw + Assert-True ($setupText -notmatch 'Read-Host|PromptForChoice|ReadLine') ` + 'Setup must fail with instructions rather than wait for input.' + $moduleText = Get-Content -LiteralPath ` + (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Raw + Assert-True (($moduleText + $setupText) -notmatch '&\s*git[^ + +]*(switch|checkout|reset|clean)') ` + 'Setup must never switch a branch in a checkout that already has one.' + Assert-True ($moduleText -match 'fetch --quiet') ` + 'Setup should fetch, which cannot disturb a working tree.' + Assert-True ($moduleText -notmatch 'git -C \$RepositoryPath pull') ` + 'Setup must not pull, which can conflict or lose work.' + Assert-True ($moduleText -match 'worktree list --porcelain') ` + 'An existing worktree for the branch should be discovered, not recreated.' + Assert-True ($moduleText -match '--path-format=absolute') ` + 'Git directory lookups must be absolute, not relative to the caller.' + + [xml]$nugetConfig = Get-Content -LiteralPath ( + Join-Path (Split-Path $PSScriptRoot -Parent) 'nuget.config') + # A local build adds its feed for that build only, so inherited + # sources can stay and a developer's private feed keeps working. + Assert-True ($null -eq $nugetConfig.SelectSingleNode( + '/configuration/packageSources/clear')) ` + 'nuget.config should leave inherited package sources in place.' +} +finally { + if (Test-Path $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force + } +} + +if ($failures.Count -gt 0) { + # Write-Host, not Write-Error: this script sets ErrorActionPreference to + # Stop, which would make the first failure terminating and hide the rest. + $failures | ForEach-Object { Write-Host $_ -ForegroundColor Red } + Write-Host ("{0} local library test(s) failed." -f $failures.Count) ` + -ForegroundColor Red + exit 1 +} + +Write-Host 'Local library tests passed.' -ForegroundColor Green diff --git a/Build/LocalLibraries.psm1 b/Build/LocalLibraries.psm1 new file mode 100644 index 0000000000..a451e533fe --- /dev/null +++ b/Build/LocalLibraries.psm1 @@ -0,0 +1,547 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Ordered, and libpalaso is first on purpose: packing order is significant +# because other libraries may depend on it. Do not alphabetise these. +$script:LibraryConfig = [ordered]@{ + palaso = @{ + VersionProperty = 'SilLibPalasoVersion' + PdbRelativeDir = 'output/Debug/net462' + CachePrefixes = @( + 'sil.core', 'sil.windows', 'sil.dblbundle', 'sil.writingsystems', + 'sil.dictionary', 'sil.lift', 'sil.lexicon', 'sil.archiving', + 'sil.media', 'sil.scripture', 'sil.testutilities' + ) + EnvVar = 'LIBPALASO_PATH' + RepoDirectory = 'libpalaso' + VersionProject = 'SIL.Core/SIL.Core.csproj' + } + l10nsharp = @{ + VersionProperty = 'L10NSharpVersion' + PdbRelativeDir = 'output/Debug/net48' + CachePrefixes = @('l10nsharp') + EnvVar = 'L10NSHARP_PATH' + RepoDirectory = 'l10nsharp' + VersionProject = 'src/L10NSharp/L10NSharp.csproj' + } + lcm = @{ + VersionProperty = 'SilLcmVersion' + PdbRelativeDir = 'artifacts/Debug/net462' + CachePrefixes = @('sil.lcmodel') + EnvVar = 'LIBLCM_PATH' + RepoDirectory = 'liblcm' + VersionProject = 'src/SIL.LCModel/SIL.LCModel.csproj' + } + chorus = @{ + VersionProperty = 'SilChorusVersion' + PdbRelativeDir = 'output/Debug/net462' + CachePrefixes = @('sil.chorus') + EnvVar = 'LIBCHORUS_PATH' + RepoDirectory = 'chorus' + VersionProject = 'src/Chorus/Chorus.csproj' + } + machine = @{ + VersionProperty = 'SilMachineVersion' + PdbRelativeDir = @( + 'src/SIL.Machine/bin/Debug/netstandard2.0', + 'src/SIL.Machine.Morphology.HermitCrab/bin/Debug/netstandard2.0' + ) + CachePrefixes = @('sil.machine') + EnvVar = 'SILMACHINE_PATH' + RepoDirectory = 'machine' + # Only the projects FieldWorks uses, which avoids the native CMake + # dependencies the rest of the repository pulls in. + PackProjects = @( + 'src/SIL.Machine/SIL.Machine.csproj', + 'src/SIL.Machine.Morphology.HermitCrab/SIL.Machine.Morphology.HermitCrab.csproj' + ) + } +} + +<# +.SYNOPSIS + Returns the local NuGet feed directory for this working tree. +.DESCRIPTION + A machine-wide feed lets one working tree's build delete packages another + working tree just produced, so the feed defaults inside the repository. + LOCAL_NUGET_REPO still wins when set, for existing setups. +#> +function Get-FieldWorksLocalFeedPath { + param([string]$RepositoryRoot) + if (-not [string]::IsNullOrWhiteSpace($env:LOCAL_NUGET_REPO)) { + return $env:LOCAL_NUGET_REPO + } + return (Join-Path $RepositoryRoot '.localfeed') +} + +<# +.SYNOPSIS + Converts a git branch name into a NuGet pre-release label. +.DESCRIPTION + SemVer pre-release identifiers allow only ASCII alphanumerics and hyphens, + so branch separators such as '/' and '_' are replaced. The result is + truncated because older tooling limited the whole pre-release string. +#> +function ConvertTo-FieldWorksVersionLabel { + param([string]$BranchName) + $label = ($BranchName -replace '[^0-9A-Za-z-]', '-').Trim('-') + $label = $label -replace '-{2,}', '-' + if ([string]::IsNullOrWhiteSpace($label)) { + return 'detached' + } + if ($label.Length -gt 24) { + $label = $label.Substring(0, 24).Trim('-') + } + return $label.ToLowerInvariant() +} + +<# +.SYNOPSIS + Describes the git state of a local library checkout. +.DESCRIPTION + Returns the branch label, short commit and whether the tree is dirty. A + dirty tree cannot be identified by commit, so callers must repack instead of + trusting the version string. Untracked files count as dirty because a new + source file changes the build without changing the commit. +#> +function Get-FieldWorksLibrarySourceState { + param([string]$SourceDirectory) + + $branch = (& git -C $SourceDirectory rev-parse --abbrev-ref HEAD 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "'$SourceDirectory' is not a git checkout; cannot derive a local version." + } + $branch = "$branch".Trim() + if ($branch -eq 'HEAD') { + $branch = 'detached' + } + + $shortSha = "$(& git -C $SourceDirectory rev-parse --short=7 HEAD 2>$null)".Trim() + $status = @(& git -C $SourceDirectory status --porcelain --untracked-files=normal 2>$null) + $isDirty = $status.Count -gt 0 + + return [pscustomobject]@{ + Branch = $branch + Label = ConvertTo-FieldWorksVersionLabel -BranchName $branch + ShortSha = $shortSha + IsDirty = $isDirty + DirtyPaths = @($status | ForEach-Object { ($_ -replace '^.{2,3}', '').Trim() }) + } +} + +<# +.SYNOPSIS + Reports whether the feed already holds this exact packed version. +.DESCRIPTION + A version derived from a clean commit identifies its contents, so an + existing package for it is the same package this pack would produce. A + dirty tree reuses one version string for changing contents, so its package + can never stand in. +#> +function Test-FieldWorksPackIsCurrent { + param([string]$LocalRepository, [string]$PackVersion, [pscustomobject]$SourceState) + if ($SourceState.IsDirty) { + return $false + } + if (-not (Test-Path -LiteralPath $LocalRepository)) { + return $false + } + return @(Get-ChildItem -LiteralPath $LocalRepository ` + -Filter "*.$PackVersion.nupkg" -File -ErrorAction SilentlyContinue).Count -gt 0 +} + +<# +.SYNOPSIS + Returns the Major.Minor.Patch a library gives itself. +.DESCRIPTION + Read from the library's own GitVersion so a version bump in the library is + reflected in the local package. Libraries without GitVersion have no + VersionProject and fall back to the version FieldWorks consumes. +#> +function Get-FieldWorksLibraryCoreVersion { + param([string]$SourceDirectory, [hashtable]$LibraryEntry, [string]$FallbackVersion) + + $fallbackCore = ($FallbackVersion -split '-', 2)[0] + if (-not $LibraryEntry.Contains('VersionProject')) { + return $fallbackCore + } + $project = Join-Path $SourceDirectory $LibraryEntry.VersionProject + if (-not (Test-Path -LiteralPath $project)) { + Write-Warning ("Version project '$project' not found; using $fallbackCore.") + return $fallbackCore + } + + # -restore first: GetVersion comes from the GitVersion package, which a + # checkout that has never been built does not have yet. + $probed = & dotnet msbuild $project -restore -t:GetVersion ` + -getProperty:GitVersion_MajorMinorPatch -v:q -nologo 2>$null + $probed = @($probed | Where-Object { $_ -match '^\d+\.\d+\.\d+$' }) + if ($LASTEXITCODE -ne 0 -or $probed.Count -eq 0) { + Write-Warning ("Could not read a GitVersion version from '$project'; " + + "using $fallbackCore.") + return $fallbackCore + } + return $probed[-1].Trim() +} + +<# +.SYNOPSIS + Builds the pre-release version for a locally packed library. +.DESCRIPTION + A clean tree is identified by its commit, so repacking the same commit + reuses the cached package correctly. A dirty tree has no stable identity, + so it is labelled 'dirty' and the caller repacks every time. +#> +function Get-FieldWorksLocalPackVersion { + param([string]$CoreVersion, [pscustomobject]$SourceState) + + $core = ($CoreVersion -split '-', 2)[0] + if ($SourceState.IsDirty) { + return "$core-$($SourceState.Label).dirty" + } + return "$core-$($SourceState.Label).$($SourceState.ShortSha)" +} + +function Test-ManagedPackageName { + param([string]$Name, [string[]]$Prefixes) + $normalizedName = $Name.ToLowerInvariant() + foreach ($prefix in $Prefixes) { + if ($normalizedName -eq $prefix -or $normalizedName.StartsWith("$prefix.")) { + return $true + } + } + return $false +} + +function Test-FilesystemPackageSource { + param([string]$Source) + if ([string]::IsNullOrWhiteSpace($Source)) { + return $false + } + $uri = $null + if ([System.Uri]::TryCreate($Source, [System.UriKind]::Absolute, [ref]$uri)) { + return $uri.IsFile + } + return [System.IO.Path]::IsPathRooted($Source) +} + +function Get-SelectedPrefixes { + param([string[]]$Libraries) + $selected = if ($Libraries -and $Libraries.Count -gt 0) { + $Libraries + } + else { + @($script:LibraryConfig.Keys) + } + $prefixes = foreach ($library in $selected) { + if (-not $script:LibraryConfig.Contains($library)) { + throw "Unknown local library '$library'." + } + $script:LibraryConfig[$library].CachePrefixes + } + return @($prefixes | Sort-Object -Unique) +} + +<# +.SYNOPSIS + Removes every cached version for the selected local-library groups. +#> +function Clear-FieldWorksLibraryPackageCache { + param([string]$PackagesDirectory, [string[]]$Libraries) + if (-not (Test-Path -LiteralPath $PackagesDirectory)) { + return + } + $prefixes = Get-SelectedPrefixes -Libraries $Libraries + $packageDirectories = @(Get-ChildItem -LiteralPath $PackagesDirectory -Directory | + Where-Object { Test-ManagedPackageName -Name $_.Name -Prefixes $prefixes }) + foreach ($packageDirectory in $packageDirectories) { + Remove-Item -LiteralPath $packageDirectory.FullName -Recurse -Force + } + if ($packageDirectories.Count -gt 0) { + Write-Host ("Cleared {0} package cache folders." -f $packageDirectories.Count) ` + -ForegroundColor Yellow + } +} + +<# +.SYNOPSIS + Returns the configuration for FieldWorks-supported local libraries. +#> +function Get-FieldWorksLocalLibraryConfig { + return $script:LibraryConfig +} + +<# +.SYNOPSIS + Removes locally sourced cache entries and managed packages from a local feed. +#> +function Clear-FieldWorksLocalLibraries { + param( + [string]$PackagesDirectory, + [string]$LocalRepository, + [string[]]$Libraries + ) + + $prefixes = Get-SelectedPrefixes -Libraries $Libraries + $cacheRemovalCount = 0 + $feedRemovalCount = 0 + + if (Test-Path -LiteralPath $PackagesDirectory) { + $packageDirectories = @(Get-ChildItem -LiteralPath $PackagesDirectory -Directory | + Where-Object { Test-ManagedPackageName -Name $_.Name -Prefixes $prefixes }) + foreach ($packageDirectory in $packageDirectories) { + foreach ($versionDirectory in @(Get-ChildItem -LiteralPath $packageDirectory.FullName -Directory)) { + $metadataPath = Join-Path $versionDirectory.FullName '.nupkg.metadata' + if (-not (Test-Path -LiteralPath $metadataPath)) { + # No metadata means extraction never finished, so nothing here can + # be trusted and restore will replace it. + Remove-Item -LiteralPath $versionDirectory.FullName -Recurse -Force + $cacheRemovalCount++ + continue + } + $source = $null + try { + $metadata = Get-Content -LiteralPath $metadataPath -Raw | ConvertFrom-Json + # Version 1 metadata records no source; reading it under + # Set-StrictMode is a terminating error, so it stays guarded. + $source = $metadata.PSObject.Properties['source'] + if ($source) { $source = [string]$source.Value } + } + catch { + Write-Warning "Could not read NuGet metadata at '$metadataPath'; preserving it." + continue + } + if (Test-FilesystemPackageSource -Source $source) { + Remove-Item -LiteralPath $versionDirectory.FullName -Recurse -Force + $cacheRemovalCount++ + } + } + if (@(Get-ChildItem -LiteralPath $packageDirectory.FullName -Force).Count -eq 0) { + Remove-Item -LiteralPath $packageDirectory.FullName -Force + } + } + } + + if ($LocalRepository -and (Test-Path -LiteralPath $LocalRepository)) { + $feedPackages = @(Get-ChildItem -LiteralPath $LocalRepository -File | + Where-Object { + $_.Extension -in @('.nupkg', '.snupkg') -and + (Test-ManagedPackageName -Name $_.BaseName -Prefixes $prefixes) + }) + foreach ($feedPackage in $feedPackages) { + Remove-Item -LiteralPath $feedPackage.FullName -Force + $feedRemovalCount++ + } + } + + if ($cacheRemovalCount -gt 0 -or $feedRemovalCount -gt 0) { + Write-Host ("Cleared {0} local cache entries and {1} local feed packages." -f ` + $cacheRemovalCount, $feedRemovalCount) -ForegroundColor Yellow + } +} + +<# +.SYNOPSIS + Converts a branch name into a directory name. +.DESCRIPTION + Branch names may contain path separators, which cannot appear in a single + directory name. Unlike a version label this keeps the full name, because + the directory is how a developer recognises the worktree. +#> +function ConvertTo-FieldWorksWorktreeName { + param([string]$BranchName) + $name = $BranchName + foreach ($invalid in [System.IO.Path]::GetInvalidFileNameChars()) { + $name = $name.Replace($invalid, '-') + } + return ($name -replace '-{2,}', '-').Trim('-') +} + +<# +.SYNOPSIS + Locates a local checkout of a library. +.DESCRIPTION + Prefers an explicit path, then a sibling of the FieldWorks checkout, then + the legacy environment variable. Throws with the ways to fix it rather than + prompting, so unattended builds fail instead of waiting for input. +#> +function Resolve-FieldWorksLibraryRepo { + param([string]$Library, [string]$RepositoryRoot, [string]$ExplicitPath) + + $entry = $script:LibraryConfig[$Library] + if (-not $entry) { + throw "Unknown local library '$Library'." + } + + # Siblings sit beside the main checkout, which is not the parent of a + # worktree, so ask git where the repository itself lives. + $mainRoot = $RepositoryRoot + $commonDir = & git -C $RepositoryRoot rev-parse --path-format=absolute ` + --git-common-dir 2>$null + if ($LASTEXITCODE -eq 0 -and $commonDir) { + $mainRoot = Split-Path ($commonDir.Trim()) -Parent + } + + $candidates = [ordered]@{} + if ($ExplicitPath) { $candidates['the path given'] = $ExplicitPath } + $sibling = Join-Path (Split-Path $mainRoot -Parent) $entry.RepoDirectory + $candidates["the sibling checkout $sibling"] = $sibling + $fromEnv = [System.Environment]::GetEnvironmentVariable($entry.EnvVar) + if ($fromEnv) { $candidates["$($entry.EnvVar)"] = $fromEnv } + + foreach ($source in $candidates.Keys) { + $path = $candidates[$source] + if (Test-Path -LiteralPath (Join-Path $path '.git')) { + return [pscustomobject]@{ Path = (Resolve-Path -LiteralPath $path).Path + Source = $source } + } + } + + throw ("No git checkout of '$Library' was found. Clone it beside FieldWorks " + + "as '$sibling', or set $($entry.EnvVar), or pass an explicit path.") +} + +<# +.SYNOPSIS + Makes a worktree of a library branch available, or explains what is missing. +.DESCRIPTION + An existing worktree on the branch is used as it stands, because git refuses + to check one branch out twice and because another worktree may hold work in + progress. Otherwise a worktree is created under .tmp/worktrees. The branch is + never switched in a checkout that already has one, so no work can be lost. +#> +function Resolve-FieldWorksLibraryWorktree { + param([string]$RepositoryPath, [string]$Branch, [switch]$Create) + + & git -C $RepositoryPath fetch --quiet 2>$null | Out-Null + + # Discovery, not creation: the branch may already be checked out somewhere, + # which is both the fast path and the only path git allows. + $listing = @(& git -C $RepositoryPath worktree list --porcelain 2>$null) + $currentPath = $null + foreach ($line in $listing) { + if ($line -like 'worktree *') { $currentPath = $line.Substring(9).Trim() } + elseif ($line -eq "branch refs/heads/$Branch") { + return [pscustomobject]@{ Path = $currentPath; Created = $false } + } + } + + $hasLocal = & git -C $RepositoryPath rev-parse --verify --quiet "refs/heads/$Branch" + $hasRemote = & git -C $RepositoryPath rev-parse --verify --quiet "refs/remotes/origin/$Branch" + if (-not $hasLocal -and -not $hasRemote) { + throw ("Branch '$Branch' does not exist in $RepositoryPath, locally or on " + + "origin. Create it there first, or name an existing branch.") + } + + $target = Join-Path $RepositoryPath (Join-Path '.tmp/worktrees' ` + (ConvertTo-FieldWorksWorktreeName -BranchName $Branch)) + if (-not $Create) { + throw ("Branch '$Branch' has no worktree in $RepositoryPath. Run with " + + "-SetupLocalLibraries to create one at $target.") + } + + # Their .gitignore belongs to another team, so exclude .tmp for this clone + # only. --git-common-dir is relative to the caller unless asked otherwise. + $commonDir = & git -C $RepositoryPath rev-parse --path-format=absolute ` + --git-common-dir 2>$null + if ($LASTEXITCODE -eq 0 -and $commonDir) { + $excludePath = Join-Path $commonDir.Trim() 'info/exclude' + $excluded = if (Test-Path -LiteralPath $excludePath) { + @(Get-Content -LiteralPath $excludePath) + } else { @() } + if (-not @($excluded | Where-Object { $_.Trim() -eq '.tmp/' -or + $_.Trim() -eq '.tmp/worktrees/' })) { + Add-Content -LiteralPath $excludePath -Value '.tmp/' + Write-Host " Excluded .tmp/ for this clone only." -ForegroundColor Gray + } + } + + $addArgs = if ($hasLocal) { @($target, $Branch) } + else { @('-b', $Branch, $target, "origin/$Branch") } + & git -C $RepositoryPath worktree add @addArgs 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $target)) { + throw "Could not create a worktree for '$Branch' at $target." + } + return [pscustomobject]@{ Path = $target; Created = $true } +} + +<# +.SYNOPSIS + Describes how a checkout stands against its upstream branch. +#> +function Get-FieldWorksLibraryUpstreamState { + param([string]$WorktreePath) + $counts = & git -C $WorktreePath rev-list --left-right --count '@{upstream}...HEAD' 2>$null + if ($LASTEXITCODE -ne 0 -or -not $counts) { + return [pscustomobject]@{ HasUpstream = $false; Behind = 0; Ahead = 0 } + } + $parts = -split $counts + return [pscustomobject]@{ HasUpstream = $true + Behind = [int]$parts[0]; Ahead = [int]$parts[1] } +} + +<# +.SYNOPSIS + Writes the generated property file that selects local libraries. +.DESCRIPTION + Written to disk rather than passed on a command line because a restore + launched through Exec starts a new MSBuild process, which does not inherit + global properties. SilVersions.props imports this file, so every process + that resolves a version sees the same answer. +#> +function Write-FieldWorksLocalLibraryProps { + param([string]$Path, [hashtable]$Versions, [string]$LocalRepository) + + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add('') + $lines.Add("`t") + $lines.Add("`t") + foreach ($name in $Versions.Keys) { + $lines.Add("`t`t<$name>$($Versions[$name])") + } + if ($LocalRepository) { + $lines.Add("`t`t" + + "`$(RestoreAdditionalProjectSources);$LocalRepository" + + '') + } + $lines.Add("`t") + $lines.Add('') + + $directory = Split-Path $Path -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + Set-Content -LiteralPath $Path -Value $lines -Encoding UTF8 +} + +<# +.SYNOPSIS + Removes the generated local library property file, if it is present. +#> +function Remove-FieldWorksLocalLibraryProps { + param([string]$Path) + if (Test-Path -LiteralPath $Path) { + Remove-Item -LiteralPath $Path -Force + Write-Host 'Removed local library overrides; using published versions.' ` + -ForegroundColor Yellow + } +} + +<# +.SYNOPSIS + Returns the path of the generated local library property file. +#> +function Get-FieldWorksLocalLibraryPropsPath { + param([string]$RepositoryRoot) + return (Join-Path $RepositoryRoot 'Build/LocalLibraries.props') +} + +Export-ModuleMember -Function Get-FieldWorksLocalLibraryConfig, + Clear-FieldWorksLocalLibraries, Clear-FieldWorksLibraryPackageCache, + Get-FieldWorksLocalFeedPath, ConvertTo-FieldWorksVersionLabel, + Get-FieldWorksLibrarySourceState, Get-FieldWorksLocalPackVersion, + Get-FieldWorksLibraryCoreVersion, Test-FieldWorksPackIsCurrent, + ConvertTo-FieldWorksWorktreeName, Resolve-FieldWorksLibraryRepo, + Resolve-FieldWorksLibraryWorktree, Get-FieldWorksLibraryUpstreamState, + Write-FieldWorksLocalLibraryProps, Remove-FieldWorksLocalLibraryProps, + Get-FieldWorksLocalLibraryPropsPath diff --git a/Build/Manage-LocalLibraries.ps1 b/Build/Manage-LocalLibraries.ps1 index 2f5e7eb1e5..eeb6e89ce2 100644 --- a/Build/Manage-LocalLibraries.ps1 +++ b/Build/Manage-LocalLibraries.ps1 @@ -5,11 +5,11 @@ .DESCRIPTION Two modes of operation: - Pack mode (one or more source paths provided): - Packs local checkouts of liblcm, libpalaso, chorus, and/or machine into the - local NuGet feed using each library's own version. Detects the version - from produced packages, updates SilVersions.props to match, copies - PDBs, and clears stale cached packages. + Build pack mode (one or more source paths and -VersionOutputPath provided): + Packs local checkouts of liblcm, libpalaso, chorus, machine, and/or + L10NSharp into the local NuGet feed using each library's own version. + Detects the versions, writes them for build.ps1, copies PDBs, and clears + stale cached packages. Multiple libraries can be packed in a single call. libpalaso is always packed first (other libraries may depend on it). @@ -66,13 +66,16 @@ Sets the version in SilVersions.props (SetVersion mode). Use to revert to an upstream version. Not used in pack mode. +.PARAMETER VersionOutputPath + Path to write the packed versions to, as JSON keyed by version property. + .EXAMPLE - .\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso - Packs libpalaso, detects its version, and updates SilVersions.props. + .\build.ps1 -LocalLibraries palaso + Rebuilds libpalaso from LIBPALASO_PATH for this FieldWorks build. .EXAMPLE - .\Build\Manage-LocalLibraries.ps1 -Palaso -Chorus -ChorusPath C:\Repos\chorus - Packs libpalaso (from env var) and then chorus from the given path. + .\build.ps1 -LocalLibraries palaso,chorus + Rebuilds libpalaso and chorus from their configured paths for this build. .EXAMPLE .\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0 @@ -98,59 +101,20 @@ param( [ValidateSet('palaso', 'lcm', 'chorus', 'machine', 'l10nsharp')] [string]$Library, - [string]$Version -) + [string]$Version, -$ErrorActionPreference = "Stop" - -# --------------------------------------------------------------------------- -# Library-specific configuration -# --------------------------------------------------------------------------- + [string]$VersionOutputPath, -$LibraryConfig = @{ - palaso = @{ - VersionProperty = 'SilLibPalasoVersion' - PdbRelativeDir = 'output/Debug/net462' - CachePrefixes = @( - 'sil.core', 'sil.windows', 'sil.dblbundle', 'sil.writingsystems', - 'sil.dictionary', 'sil.lift', 'sil.lexicon', 'sil.archiving', - 'sil.media', 'sil.scripture', 'sil.testutilities' - ) - EnvVar = 'LIBPALASO_PATH' - } - lcm = @{ - VersionProperty = 'SilLcmVersion' - PdbRelativeDir = 'artifacts/Debug/net462' - CachePrefixes = @('sil.lcmodel') - EnvVar = 'LIBLCM_PATH' - } - chorus = @{ - VersionProperty = 'SilChorusVersion' - PdbRelativeDir = 'output/Debug/net462' - CachePrefixes = @('sil.chorus') - EnvVar = 'LIBCHORUS_PATH' - } - machine = @{ - VersionProperty = 'SilMachineVersion' - PdbRelativeDir = 'bin/Debug/netstandard2.0' - CachePrefixes = @('sil.machine') - EnvVar = 'SILMACHINE_PATH' - # Pack only the projects FieldWorks uses (avoids native CMake deps) - PackProjects = @( - 'src/SIL.Machine/SIL.Machine.csproj', - 'src/SIL.Machine.Morphology.HermitCrab/SIL.Machine.Morphology.HermitCrab.csproj' - ) - } - l10nsharp = @{ - VersionProperty = 'L10NSharpVersion' - PdbRelativeDir = 'output/Debug/net462' - CachePrefixes = @('l10nsharp') - EnvVar = 'L10NSHARP_PATH' - } -} + [string]$LocalFeedPath +) -# Pack order: libpalaso first (other libraries may depend on it) -$PackOrder = @('palaso', 'l10nsharp', 'lcm', 'chorus', 'machine') +$ErrorActionPreference = "Stop" +Import-Module (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Force +$LibraryConfig = Get-FieldWorksLocalLibraryConfig +# Pack order: libpalaso first, because other libraries may depend on it. The +# order comes from the catalogue's declaration order, so do not alphabetise it. +$PackOrder = @($LibraryConfig.Keys) +$packedVersions = [ordered]@{} # --------------------------------------------------------------------------- # Read SilVersions.props @@ -205,34 +169,49 @@ function Update-VersionAndClearCache { Write-Host "Updated SilVersions.props ($($cfg.VersionProperty) = $NewVersion)" -ForegroundColor Yellow - $packagesDir = Join-Path $repoRoot "packages" - if (Test-Path $packagesDir) { - $patterns = $cfg.CachePrefixes | ForEach-Object { "$packagesDir/$_*" } - $stale = @(Get-ChildItem -Path $patterns -Directory -ErrorAction SilentlyContinue) - if ($stale.Count -gt 0) { - $stale | Remove-Item -Recurse -Force - Write-Host "Cleared $($stale.Count) stale package folder(s) from packages/." -ForegroundColor Yellow - } - } + Clear-FieldWorksLibraryPackageCache -PackagesDirectory (Join-Path $repoRoot 'packages') ` + -Libraries @($LibName) } -# --------------------------------------------------------------------------- -# Helper: extract version from a .nupkg filename -# --------------------------------------------------------------------------- -# Split on '.', find the first segment starting with a digit — everything -# from there onward (minus .nupkg) is the version. -# E.g. SIL.Windows.Forms.Keyboarding.18.0.0-beta.nupkg → 18.0.0-beta - -function Get-PackageVersion { - param([string]$FileName) - $base = $FileName -replace '\.nupkg$', '' - $segments = $base -split '\.' - for ($i = 0; $i -lt $segments.Count; $i++) { - if ($segments[$i] -match '^\d') { - return ($segments[$i..($segments.Count - 1)] -join '.') +# --- Copy a library's PDBs next to the build output --- + +function Copy-LibrarySymbols { + param([string]$LibName, [string]$SourceDir) + $cfg = $LibraryConfig[$LibName] + + $outputDebugDir = Join-Path $repoRoot "Output/Debug" + $downloadsDir = Join-Path $repoRoot "Downloads" + $copied = 0 + + # A library that writes per-project output needs one directory per project. + foreach ($relativeDir in @($cfg.PdbRelativeDir)) { + $pdbSourceDir = Join-Path $SourceDir $relativeDir + if (-not (Test-Path $pdbSourceDir)) { + continue + } + $pdbFiles = @(Get-ChildItem -Path $pdbSourceDir -Filter "*.pdb" -File) + if ($pdbFiles.Count -eq 0) { + continue + } + foreach ($dir in @($outputDebugDir, $downloadsDir)) { + if (-not (Test-Path $dir)) { + New-Item -Path $dir -ItemType Directory -Force | Out-Null + } } + $pdbFiles | Copy-Item -Destination $outputDebugDir -Force + $pdbFiles | Copy-Item -Destination $downloadsDir -Force + $copied += $pdbFiles.Count + } + + if ($copied -gt 0) { + Write-Host "Copied $copied PDB file(s) to Output/Debug/ and Downloads/..." ` + -ForegroundColor Cyan + } + else { + # Say where we looked: a wrong directory here fails silently otherwise. + Write-Host ("No PDB files found for {0} under: {1}" -f $LibName, + ((@($cfg.PdbRelativeDir)) -join ', ')) -ForegroundColor Yellow } - return $null } # --------------------------------------------------------------------------- @@ -245,109 +224,108 @@ function Invoke-PackLibrary { $cfg = $LibraryConfig[$LibName] $node = Get-VersionNode $LibName + $packTargets = if ($cfg.PackProjects -and $cfg.PackProjects.Count -gt 0) { + @($cfg.PackProjects | ForEach-Object { Join-Path $SourceDir $_ }) + } else { @($SourceDir) } + + # NuGet resolves an extracted (id, version) before consulting a folder feed, + # so a local pack must never reuse the published version string. + $sourceState = Get-FieldWorksLibrarySourceState -SourceDirectory $SourceDir + $coreVersion = Get-FieldWorksLibraryCoreVersion -SourceDirectory $SourceDir ` + -LibraryEntry $cfg -FallbackVersion $node.InnerText.Trim() + $packVersion = Get-FieldWorksLocalPackVersion ` + -CoreVersion $coreVersion -SourceState $sourceState + Write-Host "" Write-Host "========================================" -ForegroundColor Cyan Write-Host "Packing $LibName" -ForegroundColor Cyan Write-Host " Source: $SourceDir" -ForegroundColor Cyan + $dirtyNote = if ($sourceState.IsDirty) { ' (dirty)' } else { '' } + Write-Host " Branch: $($sourceState.Branch)$dirtyNote" -ForegroundColor Cyan + Write-Host " Commit: $($sourceState.ShortSha)" -ForegroundColor Cyan Write-Host " Current: $($node.InnerText.Trim())" -ForegroundColor Cyan + Write-Host " Packing as: $packVersion" -ForegroundColor Cyan Write-Host " Output: $LocalRepo" -ForegroundColor Cyan Write-Host "========================================" -ForegroundColor Cyan - # Record timestamp before pack so we can find newly-produced packages - $packStart = Get-Date + if ($sourceState.IsDirty) { + # Name the paths: while the tree is dirty every build repacks, so the + # way to a reusable package is to commit or remove these. + $shown = @($sourceState.DirtyPaths | Select-Object -First 5) + Write-Host ("Uncommitted changes force a repack ({0}):" -f + $sourceState.DirtyPaths.Count) -ForegroundColor Yellow + $shown | ForEach-Object { Write-Host " $_" -ForegroundColor Yellow } + if ($sourceState.DirtyPaths.Count -gt $shown.Count) { + Write-Host (" ... and {0} more" -f + ($sourceState.DirtyPaths.Count - $shown.Count)) -ForegroundColor Yellow + } + } + elseif (Test-FieldWorksPackIsCurrent -LocalRepository $LocalRepo ` + -PackVersion $packVersion -SourceState $sourceState) { + Write-Host "Reusing the packed $packVersion (commit unchanged)." ` + -ForegroundColor Green + $script:packedVersions[$cfg.VersionProperty] = $packVersion + Copy-LibrarySymbols -LibName $LibName -SourceDir $SourceDir + return + } + + # Only the library being repacked loses its local packages. + Clear-FieldWorksLocalLibraries -PackagesDirectory (Join-Path $repoRoot 'packages') ` + -LocalRepository $LocalRepo -Libraries @($LibName) + + # Build first: a package may include output from a framework its own project + # does not build, which pack alone does not produce. + $commonBuildArgs = @( + '-c', 'Debug' + "-p:Version=$packVersion" + '-p:DisableGitVersionTask=true' + "-p:RestoreAdditionalProjectSources=$LocalRepo" + ) + Write-Host "Running dotnet build..." -ForegroundColor Cyan + foreach ($buildTarget in $packTargets) { + & dotnet build $buildTarget @commonBuildArgs + if ($LASTEXITCODE -ne 0) { + throw "dotnet build failed for $LibName ($buildTarget)." + } + } Write-Host "Running dotnet pack..." -ForegroundColor Cyan $commonPackArgs = @( '-c', 'Debug' + "-p:Version=$packVersion" + # GitVersion.MsBuild assigns Version inside a target, which outranks a + # command-line property, so it must be switched off for the stamp to hold. + '-p:DisableGitVersionTask=true' "-p:IncludeSymbols=true" "-p:SymbolPackageFormat=snupkg" + "-p:RestoreAdditionalProjectSources=$LocalRepo" '--output', $LocalRepo ) - $projects = $cfg.PackProjects - if ($projects -and $projects.Count -gt 0) { - foreach ($proj in $projects) { - $projPath = Join-Path $SourceDir $proj - & dotnet pack $projPath @commonPackArgs - if ($LASTEXITCODE -ne 0) { - throw "dotnet pack failed for $LibName ($proj)." - } - } - } - else { - & dotnet pack $SourceDir @commonPackArgs + foreach ($packTarget in $packTargets) { + & dotnet pack $packTarget @commonPackArgs if ($LASTEXITCODE -ne 0) { - throw "dotnet pack failed for $LibName." + throw "dotnet pack failed for $LibName ($packTarget)." } } - # Find .nupkg files created after pack started (exclude .snupkg and test pkgs) - $newPackages = @( - Get-ChildItem -Path $LocalRepo -Filter "*.nupkg" -File | - Where-Object { $_.LastWriteTime -ge $packStart -and $_.Extension -eq '.nupkg' -and $_.Name -notmatch 'tests' } + # Verify the stamped version actually reached the feed. + $produced = @( + Get-ChildItem -Path $LocalRepo -Filter "*.$packVersion.nupkg" -File | + Where-Object { $_.Name -notmatch 'tests' } ) - - if ($newPackages.Count -eq 0) { - $currentVer = (Get-VersionNode $LibName).InnerText.Trim() - Write-Host "" - Write-Host "WARNING: No new .nupkg files were produced for $LibName." -ForegroundColor Yellow - Write-Host " The library version may not have changed since the last pack." -ForegroundColor Yellow - Write-Host " Current version in SilVersions.props: $currentVer" -ForegroundColor Yellow - Write-Host " Skipping version update for $LibName." -ForegroundColor Yellow - return - } - - Write-Host "New packages found:" -ForegroundColor Gray - $newPackages | ForEach-Object { Write-Host " $($_.Name)" -ForegroundColor Gray } - - $detectedVersions = @($newPackages | ForEach-Object { Get-PackageVersion $_.Name } | - Where-Object { $_ } | Sort-Object -Unique) - - Write-Host "Detected version(s): $($detectedVersions -join ', ')" -ForegroundColor Gray - - if ($detectedVersions.Count -eq 0) { - throw "Could not parse version from produced packages: $($newPackages.Name -join ', ')" - } - if ($detectedVersions.Count -gt 1) { - Write-Host "WARNING: Multiple versions detected in produced packages:" -ForegroundColor Red - $detectedVersions | ForEach-Object { Write-Host " $_" -ForegroundColor Red } - throw "Expected all packages to share one version. Clean $LocalRepo and retry." + if ($produced.Count -eq 0) { + throw ("dotnet pack produced no package for $LibName at version " + + "$packVersion. Inspect $LocalRepo.") } - $packVersion = $detectedVersions[0] Write-Host "" - Write-Host "Pack complete ($($newPackages.Count) package(s), version $packVersion)." -ForegroundColor Green + Write-Host "Pack complete ($($produced.Count) package(s), version $packVersion)." ` + -ForegroundColor Green - # Update SilVersions.props and clear cache - Update-VersionAndClearCache -LibName $LibName -NewVersion $packVersion - Write-Host "To revert: git checkout Build/SilVersions.props" -ForegroundColor Yellow + $script:packedVersions[$cfg.VersionProperty] = $packVersion - # Copy PDB files to Output/Debug/ and Downloads/ - $pdbSourceDir = Join-Path $SourceDir $cfg.PdbRelativeDir - - if (Test-Path $pdbSourceDir) { - $outputDebugDir = Join-Path $repoRoot "Output/Debug" - $downloadsDir = Join-Path $repoRoot "Downloads" - - foreach ($dir in @($outputDebugDir, $downloadsDir)) { - if (-not (Test-Path $dir)) { - New-Item -Path $dir -ItemType Directory -Force | Out-Null - } - } - - $pdbFiles = @(Get-ChildItem -Path $pdbSourceDir -Filter "*.pdb" -File) - if ($pdbFiles.Count -gt 0) { - Write-Host "Copying $($pdbFiles.Count) PDB file(s) to Output/Debug/ and Downloads/..." -ForegroundColor Cyan - $pdbFiles | Copy-Item -Destination $outputDebugDir -Force - $pdbFiles | Copy-Item -Destination $downloadsDir -Force - } - else { - Write-Host "No PDB files found in $pdbSourceDir" -ForegroundColor Yellow - } - } - else { - Write-Host "PDB source directory not found: $pdbSourceDir (PDBs will only be in .snupkg)" -ForegroundColor Yellow - } + Copy-LibrarySymbols -LibName $LibName -SourceDir $SourceDir Write-Host "" Write-Host "[OK] $LibName packed successfully." -ForegroundColor Green @@ -397,37 +375,39 @@ if ($toPack.Count -gt 0) { if ($Version) { Write-Host "WARNING: -Version is ignored in pack mode (version is detected from produced packages)." -ForegroundColor Yellow } + if (-not $VersionOutputPath) { + throw "Packing local libraries is build-scoped. Run .\build.ps1 -LocalLibraries ." + } - $localRepo = $env:LOCAL_NUGET_REPO + $localRepo = $LocalFeedPath if (-not $localRepo) { - throw "The LOCAL_NUGET_REPO environment variable is not set. Set it to a folder path (e.g. C:\localnugetpackages)." + $localRepo = Get-FieldWorksLocalFeedPath -RepositoryRoot $repoRoot } if (-not (Test-Path $localRepo)) { Write-Host "Creating local NuGet repo folder: $localRepo" -ForegroundColor Yellow New-Item -Path $localRepo -ItemType Directory -Force | Out-Null } - - # Ensure local NuGet source is registered (user-level config) - $sourceList = & dotnet nuget list source 2>&1 - $normalizedRepo = [System.IO.Path]::GetFullPath($localRepo).TrimEnd('\', '/') - $alreadyRegistered = $sourceList | Where-Object { - $_.Trim() -replace '[\\/]$', '' -ieq $normalizedRepo - } - if (-not $alreadyRegistered) { - & dotnet nuget add source $localRepo --name local 2>&1 | Out-Null - Write-Host "Added local NuGet source: $localRepo" -ForegroundColor Yellow - } - Write-Host "" Write-Host "Libraries to pack: $($toPack.Keys -join ', ')" -ForegroundColor Cyan foreach ($lib in $toPack.Keys) { Invoke-PackLibrary -LibName $lib -SourceDir $toPack[$lib] -LocalRepo $localRepo } + $versionOutputDirectory = Split-Path $VersionOutputPath -Parent + if ($versionOutputDirectory -and -not (Test-Path $versionOutputDirectory)) { + New-Item -Path $versionOutputDirectory -ItemType Directory -Force | Out-Null + } + $packedVersions | ConvertTo-Json | Set-Content -LiteralPath $VersionOutputPath ` + -Encoding UTF8 + + # The property file, not the command line, is what a nested restore can see. + Write-FieldWorksLocalLibraryProps ` + -Path (Get-FieldWorksLocalLibraryPropsPath -RepositoryRoot $repoRoot) ` + -Versions $packedVersions -LocalRepository $localRepo Write-Host "" Write-Host "========================================" -ForegroundColor Green - Write-Host "[OK] All libraries packed. Run .\build.ps1 to build." -ForegroundColor Green + Write-Host "[OK] Selected local libraries packed for this build." -ForegroundColor Green Write-Host "========================================" -ForegroundColor Green } elseif ($Library -and $Version) { @@ -450,5 +430,5 @@ elseif ($Library -and $Version) { Write-Host "Run .\build.ps1 to restore and build with the new version." -ForegroundColor Cyan } else { - throw "Nothing to do. Use -Palaso/-Lcm/-Chorus/-Machine/-L10nSharp switches to pack, or -Library and -Version to set a version.`nExamples:`n .\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso`n .\Build\Manage-LocalLibraries.ps1 -Palaso -Chorus`n .\Build\Manage-LocalLibraries.ps1 -Machine -MachinePath C:\Repos\machine`n .\Build\Manage-LocalLibraries.ps1 -Library l10nsharp -Version 10.0.0`n .\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0" + throw "Nothing to do. Use .\build.ps1 -LocalLibraries to pack local libraries, or -Library and -Version to set a version.`nExamples:`n .\build.ps1 -LocalLibraries palaso`n .\build.ps1 -LocalLibraries palaso,chorus`n .\build.ps1 -LocalLibraries machine`n .\Build\Manage-LocalLibraries.ps1 -Library l10nsharp -Version 10.0.0`n .\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0" } diff --git a/Build/Setup-LocalLibraries.ps1 b/Build/Setup-LocalLibraries.ps1 new file mode 100644 index 0000000000..99377a3ebd --- /dev/null +++ b/Build/Setup-LocalLibraries.ps1 @@ -0,0 +1,91 @@ +<# +.SYNOPSIS + Prepares local library checkouts for a FieldWorks build. + +.DESCRIPTION + Takes one or more : pairs, finds each library's git + checkout, and makes the branch available as a worktree. A branch that is + already checked out somewhere is used where it is: git refuses to check one + branch out twice, and another worktree may hold work in progress. Nothing is + ever switched in a checkout that already has a branch, so no work is lost. + + The command reports the resolved paths. It never prompts, so it behaves the + same when run unattended. + +.PARAMETER Library + One or more : pairs, for example lcm:my-fix. + +.PARAMETER Path + Optional explicit checkout path, valid when a single library is named. + +.EXAMPLE + .\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix + Makes the liblcm branch my-fix available and reports its path. + +.EXAMPLE + .\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix,palaso:my-fix + Prepares two libraries in one call. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string[]]$Library, + + [string]$Path +) + +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'LocalLibraries.psm1') -Force + +$repoRoot = Split-Path $PSScriptRoot -Parent +if ($Path -and $Library.Count -gt 1) { + throw '-Path applies to a single library; name one library or drop -Path.' +} + +$resolved = [ordered]@{} +foreach ($request in $Library) { + $parts = $request -split ':', 2 + if ($parts.Count -ne 2 -or -not $parts[1]) { + throw "Expected : but got '$request' (for example lcm:my-fix)." + } + $name = $parts[0].Trim() + $branch = $parts[1].Trim() + + $repo = Resolve-FieldWorksLibraryRepo -Library $name -RepositoryRoot $repoRoot ` + -ExplicitPath $Path + Write-Host "" + Write-Host "$name" -ForegroundColor Cyan + Write-Host " Checkout: $($repo.Path) (from $($repo.Source))" -ForegroundColor Gray + + $worktree = Resolve-FieldWorksLibraryWorktree -RepositoryPath $repo.Path ` + -Branch $branch -Create + $verb = if ($worktree.Created) { 'created' } else { 'already present' } + Write-Host " Branch: $branch ($verb)" -ForegroundColor Gray + Write-Host " Worktree: $($worktree.Path)" -ForegroundColor Green + + $upstream = Get-FieldWorksLibraryUpstreamState -WorktreePath $worktree.Path + if ($upstream.HasUpstream -and ($upstream.Behind -gt 0 -or $upstream.Ahead -gt 0)) { + # Reported, not merged: a fetch cannot disturb the working tree but a + # merge can, and this worktree may not be the one the caller is editing. + Write-Host (" Upstream: behind {0}, ahead {1} — reconcile it yourself" -f + $upstream.Behind, $upstream.Ahead) -ForegroundColor Yellow + } + + $state = Get-FieldWorksLibrarySourceState -SourceDirectory $worktree.Path + if ($state.IsDirty) { + Write-Host (" Note: {0} uncommitted change(s); builds will repack" -f + $state.DirtyPaths.Count) -ForegroundColor Yellow + } + + $resolved[$name] = $worktree.Path +} + +Write-Host "" +Write-Host "Point this build at the worktrees:" -ForegroundColor Cyan +foreach ($name in $resolved.Keys) { + $envVar = (Get-FieldWorksLocalLibraryConfig)[$name].EnvVar + Write-Host (" `$env:{0} = '{1}'" -f $envVar, $resolved[$name]) +} +Write-Host "" +Write-Host ("Then build with: .\build.ps1 -LocalLibraries {0}" -f + ($resolved.Keys -join ',')) -ForegroundColor Cyan diff --git a/Build/SilVersions.props b/Build/SilVersions.props index 551972abe1..8a5c898621 100644 --- a/Build/SilVersions.props +++ b/Build/SilVersions.props @@ -7,6 +7,7 @@ Imported by: - Directory.Packages.props (managed NuGet CPM) - Build/mkall.targets (native C++ build) + - Build/PackageRestore.targets (restore, including nested ones) Update a version here to update ALL consumers automatically. ============================================================= @@ -24,4 +25,12 @@ 70.1.152 60.0.56 + + + diff --git a/Docs/architecture/dependencies.md b/Docs/architecture/dependencies.md index 140ae06601..58fed1ccae 100644 --- a/Docs/architecture/dependencies.md +++ b/Docs/architecture/dependencies.md @@ -34,14 +34,16 @@ By default, dependencies are downloaded as NuGet packages during the build. The ## Building and Debugging Dependencies Locally -If you need to debug into or modify a dependency library, use the `Build/Manage-LocalLibraries.ps1` script. It packs a local checkout into a local NuGet feed, detects the produced version, and updates `SilVersions.props` to match. +If you need to debug into or modify a dependency library, select it with +`build.ps1 -LocalLibraries`. The build packs the local checkout into a local NuGet +feed inside this working tree and uses its derived version for that invocation +without changing `SilVersions.props`. Quick start: ```powershell -$env:LOCAL_NUGET_REPO = "C:\localnugetpackages" -.\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso -.\build.ps1 +$env:LIBPALASO_PATH = "C:\Repos\libpalaso" +.\build.ps1 -LocalLibraries palaso ``` For the full workflow (setup, pack, build, debug, revert), see **[Local Library Debugging](local-library-debugging.md)**. diff --git a/Docs/architecture/local-library-debugging.md b/Docs/architecture/local-library-debugging.md index 02b2fba90d..3621d408f6 100644 --- a/Docs/architecture/local-library-debugging.md +++ b/Docs/architecture/local-library-debugging.md @@ -1,149 +1,140 @@ # Local Library Debugging -This document describes how to debug locally-modified versions of **liblcm**, **libpalaso**, **chorus**, or **machine** (SIL.Machine) in FieldWorks using a local NuGet feed. +Use `build.ps1 -LocalLibraries` to rebuild locally modified SIL libraries and +use them for one FieldWorks build. A later build that does not select a library +automatically removes its local packages and restores the published version. -## Overview +## One-time setup -The workflow uses a single PowerShell script (`Build/Manage-LocalLibraries.ps1`) that: +Packed packages go to `.localfeed` inside this working tree, so no feed setup is +needed. The feed is per working tree on purpose: a machine-wide folder lets one +working tree's build delete packages another working tree just produced. +`LOCAL_NUGET_REPO` still overrides the location when it is set, but sharing one +folder across working trees reintroduces that problem. -1. Adds a local NuGet source to `nuget.config` (pointing to your `LOCAL_NUGET_REPO` folder). -2. Runs `dotnet pack` in Debug configuration with symbols, letting the library use its own version. -3. Detects the version from the produced packages. -4. Updates `SilVersions.props` so FieldWorks resolves that exact version. -5. Places `.nupkg` / `.snupkg` in your local NuGet feed folder. -6. Copies PDB files to `Output/Debug/` and `Downloads/` for debugger access. -7. Clears stale cached packages so the next restore picks up the local build. +## Prepare a library branch -This approach works identically for all three libraries. - -## Setup (one-time) - -### 1. Create a local NuGet folder - -Pick any folder, for example: - -``` -C:\localnugetpackages -``` - -### 2. Set the `LOCAL_NUGET_REPO` environment variable +`Setup-LocalLibraries.ps1` finds a library's checkout and makes a branch +available as a worktree: ```powershell -# Current session -$env:LOCAL_NUGET_REPO = "C:\localnugetpackages" - -# Persistent (user-level) -[System.Environment]::SetEnvironmentVariable("LOCAL_NUGET_REPO", "C:\localnugetpackages", "User") +.\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix +.\Build\Setup-LocalLibraries.ps1 -Library lcm:my-fix,palaso:my-fix ``` -The script automatically registers this folder as a NuGet source in your user-level NuGet config when you pack. The repo's `nuget.config` is not modified. - -### 3. Clone the library you need - -```powershell -git clone https://github.com/sillsdev/liblcm.git -git clone https://github.com/sillsdev/libpalaso.git -git clone https://github.com/sillsdev/chorus.git -git clone https://github.com/sillsdev/machine.git -``` - -## Pack a local library - -```powershell -# Single library — explicit path -.\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso +It looks for the checkout in the path you pass, then beside the FieldWorks +checkout (`../liblcm`), then in the library's path variable. If the branch is +already checked out somewhere it uses that worktree as it stands, because git +refuses to check one branch out twice and because that worktree may hold work in +progress. Otherwise it creates one under the library's `.tmp/worktrees/`. -# Multiple libraries (libpalaso is always packed first) -.\Build\Manage-LocalLibraries.ps1 -Palaso -PalasoPath C:\Repos\libpalaso -Chorus -ChorusPath C:\Repos\chorus -``` +The command fetches but never merges, and never switches a branch in a checkout +that already has one, so it cannot disturb work you have not committed. It +reports what it found and never waits for input, so it behaves the same when run +unattended. It prints the path variable to set for the worktree it resolved. -Or set environment variables so you can omit the paths: +Set the path variable for each local checkout you use: ```powershell $env:LIBPALASO_PATH = "C:\Repos\libpalaso" $env:LIBLCM_PATH = "C:\Repos\liblcm" $env:LIBCHORUS_PATH = "C:\Repos\chorus" $env:SILMACHINE_PATH = "C:\Repos\machine" - -# Switches still required — env vars only provide the path -.\Build\Manage-LocalLibraries.ps1 -Palaso -Chorus +$env:L10NSHARP_PATH = "C:\Repos\L10NSharp" ``` -The script: -- Lets the library build with its own version (no version override). -- Detects the produced version and updates `Build/SilVersions.props` to match. -- Produces `.snupkg` symbol packages (same format as production). -- Copies PDB files to `Output/Debug/` and `Downloads/` for the debugger. -- Clears stale packages from the `packages/` cache. +## Build with local libraries -## Build FieldWorks +Name every local library that this invocation should use: ```powershell -.\build.ps1 -``` +# Rebuild and use Machine locally for this build. +.\build.ps1 -LocalLibraries machine -The build will print a yellow message listing any local packages detected in `LOCAL_NUGET_REPO`. NuGet restore will use your local packages because `SilVersions.props` was updated to request the exact version produced by the library. +# Rebuild and use Palaso and Chorus locally for this build. +.\build.ps1 -LocalLibraries palaso,chorus +``` -## Debug +Every selected library is repacked from its configured checkout. The packed +versions are passed to restore and MSBuild without modifying +`Build/SilVersions.props`. The local feed is a restore source only for that +invocation. -1. Open FieldWorks in Visual Studio. -2. PDB files are already in `Output/Debug/` — the debugger will find them automatically. -3. If breakpoints show "No symbols loaded", disable **Debug > Options > Enable Just My Code**. -4. You can also open the library solution side-by-side and use **Debug > Attach to Process**. +Each pack is versioned from the checkout it came from, as +`-.` — for example `3.9.2-docs-hc-llm-guide.9358825`. This +matters because NuGet resolves an already-extracted `(id, version)` in +`packages/` before it consults a folder feed: if a local pack reused the +published version string, restore would silently serve whichever copy landed +first. Deriving the version from the branch and commit makes the two +distinguishable, and makes repacking the same commit a correct cache hit. -## Iterating +Each selected library is built before it is packed, because a package may +include output from a target framework its own project does not build. -After each change to the library: +The core comes from the library's own GitVersion where it has one, so a version +bump in the library is reflected in the local package. A library without +GitVersion, such as SIL.Machine, falls back to the version FieldWorks consumes. -1. Re-run `Manage-LocalLibraries.ps1` (~30-60 seconds). -2. Re-run `.\build.ps1`. +Because a clean commit identifies its contents, a second build from the same +commit reuses the package already in the feed instead of repacking it. -## Setting a specific version +A checkout with uncommitted changes has no stable identity, so it is packed as +`-.dirty` and repacked on every build. The build lists the paths +responsible, because committing or removing them is what restores the faster +path. Untracked files count, which means anything `.gitignore` does not cover: +a stray note beside the source is enough to keep repacking. -Use `-Version` to set any library to a specific version in `SilVersions.props` without packing: +An ordinary build selects no local libraries: ```powershell -# Revert libpalaso to an upstream version -.\Build\Manage-LocalLibraries.ps1 -Library libpalaso -Version 17.0.0 - -# Set liblcm to a specific pre-release version -.\Build\Manage-LocalLibraries.ps1 -Library liblcm -Version 11.0.0-beta0159 +.\build.ps1 ``` -This updates `SilVersions.props` and clears stale cached packages. Run `.\build.ps1` afterward to restore and build with the new version. +Before restore, every build removes managed packages from the local feed and +removes managed cache versions whose NuGet metadata identifies a filesystem +source. Published packages restored from an HTTP source stay cached, so an +ordinary build does not redownload them every time. Package sources inherited +from user-level configuration are left alone, because a local build adds its own +feed for that build only. -## Reverting to upstream packages +The selected versions and that feed are written to a generated +`Build/LocalLibraries.props`, which `Build/SilVersions.props` imports. They are +written to a file rather than passed on the command line because a restore +launched through `Exec` is a new MSBuild process and does not inherit global +properties, so a version passed that way would not reach it. -Use `-Version` to set the library back to its upstream version: +## Debug and iterate -```powershell -.\Build\Manage-LocalLibraries.ps1 -Library libpalaso -Version 17.0.0 -``` +The local build copies PDBs to `Output/Debug/` and `Downloads/`. If Visual +Studio reports that symbols are not loaded, disable **Debug > Options > Enable +Just My Code**. -Or revert all libraries at once: +After each local-library change, rerun `build.ps1` with the same +`-LocalLibraries` selection. Omit a library whenever FieldWorks should return to +its published package. -```powershell -git checkout Build/SilVersions.props -Remove-Item -Recurse packages/sil.* -.\build.ps1 -``` +## Set an explicit published version -To also remove the user-level local source: +The lower-level script still supports changing `SilVersions.props` deliberately: ```powershell -dotnet nuget remove source local +.\Build\Manage-LocalLibraries.ps1 -Library palaso -Version 17.0.0 ``` +Local packing through `Manage-LocalLibraries.ps1` is build-internal. Run +`build.ps1 -LocalLibraries` instead. + ## Supported libraries -| Library | Switch | Path parameter | Version property | Env var fallback | -|---------|--------|---------------|------------------|-----------------| -| liblcm | `-Lcm` | `-LcmPath` | `SilLcmVersion` | `LIBLCM_PATH` | -| libpalaso | `-Palaso` | `-PalasoPath` | `SilLibPalasoVersion` | `LIBPALASO_PATH` | -| chorus | `-Chorus` | `-ChorusPath` | `SilChorusVersion` | `LIBCHORUS_PATH` | -| machine | `-Machine` | `-MachinePath` | `SilMachineVersion` | `SILMACHINE_PATH` | +| Library | Selection | Version property | Checkout environment variable | +|---------|-----------|------------------|-------------------------------| +| libpalaso | `palaso` | `SilLibPalasoVersion` | `LIBPALASO_PATH` | +| L10NSharp | `l10nsharp` | `L10NSharpVersion` | `L10NSHARP_PATH` | +| liblcm | `lcm` | `SilLcmVersion` | `LIBLCM_PATH` | +| chorus | `chorus` | `SilChorusVersion` | `LIBCHORUS_PATH` | +| Machine | `machine` | `SilMachineVersion` | `SILMACHINE_PATH` | -## See Also +## See also -- [Dependencies](dependencies.md) — overview of external dependencies -- [Build Instructions](../../.github/instructions/build.instructions.md) — building FieldWorks +- [Dependencies](dependencies.md) +- [Build Instructions](../../.github/instructions/build.instructions.md) diff --git a/build.ps1 b/build.ps1 index 3ff6eaa985..5b5783802f 100644 --- a/build.ps1 +++ b/build.ps1 @@ -114,6 +114,10 @@ Path to the local liblcm repository. Defaults to ../liblcm relative to the FieldWorks repo root. Only used when -UseLocalLcm is specified. +.PARAMETER LocalLibraries + Local SIL libraries to rebuild and use for this invocation. Supported values are + palaso, lcm, chorus, machine, and l10nsharp. Omitted libraries are cleaned before restore. + .PARAMETER StartedBy Optional actor label written to the worktree lock metadata (for example: user or agent). Defaults to the FW_BUILD_STARTED_BY environment variable when set, otherwise 'unknown'. @@ -153,6 +157,10 @@ .\build.ps1 -UseLocalLcm Builds FieldWorks, then builds liblcm from ../liblcm and copies DLLs into Output. +.EXAMPLE + .\build.ps1 -LocalLibraries machine + Rebuilds Machine from SILMACHINE_PATH and uses it only for this build. + .NOTES FieldWorks is x64-only. The x86 platform is no longer supported. #> @@ -191,6 +199,8 @@ param( [switch]$EnableTracing, [switch]$UseLocalLcm, [string]$LocalLcmPath, + [ValidateSet('palaso', 'lcm', 'chorus', 'machine', 'l10nsharp')] + [string[]]$LocalLibraries = @(), [ValidateSet('user', 'agent', 'unknown')] [string]$StartedBy = 'unknown', [switch]$SkipWorktreeLock, @@ -575,6 +585,58 @@ try { & $staleDllScript -OutputDir $outputDir -RepoRoot $PSScriptRoot -Verbose:$VerbosePreference } + $localLibrariesModule = Join-Path $PSScriptRoot 'Build/LocalLibraries.psm1' + Import-Module $localLibrariesModule -Force + $packagesDir = Join-Path $PSScriptRoot 'packages' + $localFeed = Get-FieldWorksLocalFeedPath -RepositoryRoot $PSScriptRoot + $localPropsPath = Get-FieldWorksLocalLibraryPropsPath -RepositoryRoot $PSScriptRoot + + # Only when this build will restore. Removing packages that nothing is + # going to put back would leave the tree unbuildable. + if (-not $SkipRestore) { + Clear-FieldWorksLocalLibraries -PackagesDirectory $packagesDir ` + -LocalRepository $localFeed + + # An earlier selection goes first, so a build naming no library + # resolves published versions even in a restore it does not launch. + Remove-FieldWorksLocalLibraryProps -Path $localPropsPath + } + + $localVersionProperties = [ordered]@{} + $localRestoreSourceArg = $null + if ($LocalLibraries.Count -gt 0) { + if ($SkipRestore) { + throw '-LocalLibraries cannot be combined with -SkipRestore.' + } + if ($UseLocalLcm -and $LocalLibraries -contains 'lcm') { + throw 'Choose either -LocalLibraries lcm or -UseLocalLcm, not both.' + } + $versionOutputPath = Join-Path ([System.IO.Path]::GetTempPath()) ` + ("FieldWorksLocalVersions_{0}.json" -f [System.Guid]::NewGuid().ToString('N')) + $managerArgs = @{ VersionOutputPath = $versionOutputPath + LocalFeedPath = $localFeed } + foreach ($localLibrary in $LocalLibraries) { + $managerArgs[$localLibrary] = $true + } + try { + # No $LASTEXITCODE check: that script reports failure by throwing, + # so the variable would hold the last native command's result. + & (Join-Path $PSScriptRoot 'Build/Manage-LocalLibraries.ps1') @managerArgs + $versionOutput = Get-Content -LiteralPath $versionOutputPath -Raw | + ConvertFrom-Json + foreach ($property in $versionOutput.PSObject.Properties) { + $localVersionProperties[$property.Name] = [string]$property.Value + } + } + finally { + if (Test-Path -LiteralPath $versionOutputPath) { + Remove-Item -LiteralPath $versionOutputPath -Force + } + } + $localRestoreSourceArg = + "/p:RestoreAdditionalProjectSources=$localFeed" + } + # ============================================================================= # Build Configuration # ============================================================================= @@ -616,6 +678,9 @@ try { # Properties $finalMsBuildArgs += "/p:Configuration=$Configuration" $finalMsBuildArgs += "/p:Platform=$Platform" + foreach ($propertyName in $localVersionProperties.Keys) { + $finalMsBuildArgs += "/p:$propertyName=$($localVersionProperties[$propertyName])" + } if ($SkipNative) { $finalMsBuildArgs += "/p:SkipNative=true" } @@ -657,18 +722,12 @@ try { Write-Host "Including optional FieldWorks executables" -ForegroundColor Yellow } - # Report local library packages when LOCAL_NUGET_REPO is configured - if ($env:LOCAL_NUGET_REPO -and (Test-Path $env:LOCAL_NUGET_REPO)) { - $localPkgs = Get-ChildItem -Path $env:LOCAL_NUGET_REPO -Filter "SIL.*.nupkg" -File -ErrorAction SilentlyContinue - if ($localPkgs.Count -gt 0) { - Write-Host "" - Write-Host "Local library packages detected in $($env:LOCAL_NUGET_REPO):" -ForegroundColor Yellow - foreach ($pkg in $localPkgs) { - Write-Host " $($pkg.Name)" -ForegroundColor Yellow - } - Write-Host "These will shadow upstream NuGet packages during restore." -ForegroundColor Yellow - Write-Host "" - } + if ($LocalLibraries.Count -gt 0) { + # Say how to undo it now, while the person who chose it is here. + Write-Host "To go back to published packages, build without -LocalLibraries." ` + -ForegroundColor Cyan + Write-Host "Using local libraries: $($LocalLibraries -join ', ')" ` + -ForegroundColor Yellow } # Bootstrap: Build FwBuildTasks first (required by SetupInclude.targets) @@ -705,7 +764,21 @@ try { if (-not (Test-Path $packagesDir)) { New-Item -Path $packagesDir -ItemType Directory -Force | Out-Null } - & dotnet restore "$PSScriptRoot\FieldWorks.sln" /p:NoWarn=NU1903 /p:DisableWarnForInvalidRestoreProjects=true "/p:Configuration=$Configuration" "/p:Platform=$Platform" --verbosity quiet + $restoreArgs = @( + "$PSScriptRoot\FieldWorks.sln", + '/p:NoWarn=NU1903', + '/p:DisableWarnForInvalidRestoreProjects=true', + "/p:Configuration=$Configuration", + "/p:Platform=$Platform", + '--verbosity', 'quiet' + ) + foreach ($propertyName in $localVersionProperties.Keys) { + $restoreArgs += "/p:$propertyName=$($localVersionProperties[$propertyName])" + } + if ($localRestoreSourceArg) { + $restoreArgs += $localRestoreSourceArg + } + & dotnet restore @restoreArgs if ($LASTEXITCODE -ne 0) { throw "NuGet package restore failed for FieldWorks.sln" } diff --git a/nuget.config b/nuget.config index 3d86798cf0..ce6b66f7b2 100644 --- a/nuget.config +++ b/nuget.config @@ -30,9 +30,9 @@ diff --git a/test.ps1 b/test.ps1 index 10f99419ef..54a8745acb 100644 --- a/test.ps1 +++ b/test.ps1 @@ -125,6 +125,15 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# Only on a full run: a caller who named a project or filter asked for that. +if (-not $TestProject -and -not $TestFilter) { + $localLibrariesTestPath = Join-Path $PSScriptRoot "Build/LocalLibraries.Tests.ps1" + & $localLibrariesTestPath + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } +} + if (-not $PSBoundParameters.ContainsKey('StartedBy') -and -not [string]::IsNullOrWhiteSpace($env:FW_BUILD_STARTED_BY)) { $startedByFromEnv = $env:FW_BUILD_STARTED_BY.ToLowerInvariant() if ($startedByFromEnv -in @('user', 'agent', 'unknown')) {