diff --git a/.github/scripts/appveyor/02-install.bat b/.github/scripts/appveyor/02-install.bat index 6914e4715a..fb57758bb2 100644 --- a/.github/scripts/appveyor/02-install.bat +++ b/.github/scripts/appveyor/02-install.bat @@ -45,5 +45,5 @@ REM Preserve the current working directory: set CHERE_INVOKING=yes REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 -C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; ccache -sv || echo "SKIP: Could not query ccache stats" ; ccache -o sloppiness=file_macro || true ; ccache -o compression=true || true ; ccache -o max_size=750M || true ' +C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; if ! ccache -sv ; then echo "SKIP: Could not query ccache stats" ; fi ; ccache -o compression=true ; echo "ccache settings: max_size=`ccache -k max_size` compression=`ccache -k compression`" ; true ' if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/03-before_build.bat b/.github/scripts/appveyor/03-before_build.bat index eaad237eba..fb3c9506d3 100644 --- a/.github/scripts/appveyor/03-before_build.bat +++ b/.github/scripts/appveyor/03-before_build.bat @@ -19,5 +19,9 @@ REM Preserve the current working directory: set CHERE_INVOKING=yes REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 +REM Zero the ccache statistics (02-install.bat printed the restored ones), +REM so that reports of later steps (05, 08, 12) count just this build: +C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -z ; true ' +if errorlevel 1 exit /b %ERRORLEVEL% C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; export MSYS2_PATH ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" bash ./scripts/Windows/build-mingw-prereqs.sh' if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/04-build_script.bat b/.github/scripts/appveyor/04-build_script.bat index ac2442246e..9b9aa69659 100644 --- a/.github/scripts/appveyor/04-build_script.bat +++ b/.github/scripts/appveyor/04-build_script.bat @@ -21,5 +21,9 @@ REM to find "nearby" program or configuration files (see common.c REM for current implementation). Hard-coded fallback strings may REM end up getting used in those cases. REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml -C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss timeout -k 30 "$R" ./ci_build.sh --with-docs=no' +REM The "ccache -c" runs whatever the build result: AppVeyor saves the cache of +REM a failed (or deadline-terminated) build right after the failing step, so +REM this enforces ccache max_size before that. Successful builds also get it in +REM 12-on_success.ps1 (after the later steps which compile tests and bundles). +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss timeout -k 30 "$R" ./ci_build.sh --with-docs=no ; RES=$? ; if ! PATH="/mingw64/bin:$PATH" ccache -c ; then echo "SKIP: Could not clean up ccache" ; fi ; exit "$RES" ' if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/05-after_build.bat b/.github/scripts/appveyor/05-after_build.bat index 3ebf4e15a8..fe84a2d41d 100644 --- a/.github/scripts/appveyor/05-after_build.bat +++ b/.github/scripts/appveyor/05-after_build.bat @@ -9,5 +9,5 @@ REM Preserve the current working directory: set CHERE_INVOKING=yes REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 -C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"' +C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; if ! ccache -sv ; then if ! ccache -s ; then echo "SKIP: Could not query ccache stats" ; fi ; fi' if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/07-test_script.bat b/.github/scripts/appveyor/07-test_script.bat index aa1a9179c6..86574c16f5 100644 --- a/.github/scripts/appveyor/07-test_script.bat +++ b/.github/scripts/appveyor/07-test_script.bat @@ -10,8 +10,9 @@ set CHERE_INVOKING=yes REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 REM Time limits: see CI_DEADLINE_EPOCH in appveyor.yml -REM Start Mingw-based integration and unit checks: -C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s check || bash -lc "for F in tests/*.log tests/*.trs ; do echo \"===---=== $F :\"; cat \"$F\"; done; exit 1;" ' +REM Start Mingw-based integration and unit checks (in parallel: measured locally, +REM 16 tests passed in all of 7 runs, with -j2 taking ~30% less time than serial): +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 2 check ; RES=$? ; if [ "$RES" -ne 0 ] ; then for F in tests/*.log tests/*.trs ; do echo "===---=== $F :" ; cat "$F" ; done ; exit "$RES" ; fi ' if errorlevel 1 exit /b %ERRORLEVEL% REM Start a Mingw-based documentation spellcheck (note that "make check" above could cover some documents in path of artifact delivery, but maybe not all that we have): C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 spellcheck' diff --git a/.github/scripts/appveyor/08-after_test.bat b/.github/scripts/appveyor/08-after_test.bat index a0195f2af0..900640ef61 100644 --- a/.github/scripts/appveyor/08-after_test.bat +++ b/.github/scripts/appveyor/08-after_test.bat @@ -14,19 +14,20 @@ REM Note: currently Python installation path with MSYS is buggy [#1584] REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; set -e ; if ! rm -rf ".inst" ; then echo "WARNING: Failed to clean away .inst" ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 install-win-bundle DESTDIR="`pwd`/.inst/NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%"' if errorlevel 1 exit /b %ERRORLEVEL% -C:\msys64\usr\bin\bash -lc 'date -u; set -e ; rm -rf ./.inst/NUT-for-Windows-x86_64-SNAPSHOT || true ; ln -fs "NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%" ./.inst/NUT-for-Windows-x86_64-SNAPSHOT ; ( cd .inst/NUT-for-Windows-x86_64-SNAPSHOT ; find . -ls ; )' -if errorlevel 1 exit /b %ERRORLEVEL% C:\msys64\usr\bin\bash -lc 'date -u' pushd .inst if errorlevel 1 exit /b %ERRORLEVEL% -REM 7z -snh: hoping for a version that can save hardlinks as such, and in 7z format to boot: -7z a -snh ../NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%.7z NUT* +REM 7z -snh: hoping for a version that can save hardlinks as such, and in 7z format to boot. +REM -mx=3 -mmt=on: the default level took about 4 minutes of the CI time limit +REM for this (debug build) archive alone. Measured on a 679 MB bundle with two +REM threads: default 98s/52MB, -mx=3 10s/76MB, -mx=1 12s/199MB (other method). +7z a -snh -mx=3 -mmt=on ../NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%.7z NUT* if errorlevel 1 exit /b %ERRORLEVEL% popd REM Report ccache stats (here rather than in a separate yml step after the REM artifact message in 08-after_test.ps1, the order does not matter): -C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"' +C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; if ! ccache -sv ; then if ! ccache -s ; then echo "SKIP: Could not query ccache stats" ; fi ; fi' if errorlevel 1 exit /b %ERRORLEVEL% -C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -x || echo "SKIP: Could not query ccache compression stats"' +C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; if ! ccache -x ; then echo "SKIP: Could not query ccache compression stats" ; fi' if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/12-on_success.ps1 b/.github/scripts/appveyor/12-on_success.ps1 new file mode 100644 index 0000000000..4c9de4e321 --- /dev/null +++ b/.github/scripts/appveyor/12-on_success.ps1 @@ -0,0 +1,42 @@ +# Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +# with MSYS2/MinGW x64: enforce ccache max_size before the cache save, and +# report ccache statistics of this build (counters are zeroed in +# 03-before_build.bat) in the job messages, which the GitHub notification +# template in appveyor.yml includes for passed builds. + +# Run ccache through MSYS2 bash: CCACHE_DIR from appveyor.yml is a POSIX path, +# which the MSYS2 runtime converts for ccache.exe like during the build itself. +# Keep the bash command free of double quotes (Windows PowerShell 5.1 mangles +# them in native command arguments), and print everything to stdout. +$env:CHERE_INVOKING = "yes" +$env:MSYSTEM = "MINGW64" + +try { + # AppVeyor saves the build cache right after the on_success scripts, so + # clean up first (later steps also compiled tests and bundles after the + # cleanup at the end of 04-build_script.bat); then the reported cache size + # is what gets saved. + $out = & C:\msys64\usr\bin\bash -lc 'date -u ; PATH=/mingw64/bin:$PATH ; export PATH ; if ! ccache -c ; then echo SKIP: Could not clean up ccache ; fi ; ccache --print-stats' + if ($LASTEXITCODE -ne 0) { throw "ccache --print-stats exited with code $LASTEXITCODE" } + $s = @{} + $out | ForEach-Object { + $kv = $_ -split "`t", 2 + if ($kv.Count -eq 2) { $s[$kv[0]] = [long]$kv[1] } else { echo $_ } + } + if ($s.Count -eq 0) { throw "no ccache statistics" } + + $hits = $s['direct_cache_hit'] + $s['preprocessed_cache_hit'] + $total = $hits + $s['cache_miss'] + $pct = 0 + if ($total -gt 0) { $pct = [math]::Round(100.0 * $hits / $total, 1) } + # Invariant culture: same number formatting whatever the worker locale is + $msg = [string]::Format([System.Globalization.CultureInfo]::InvariantCulture, + "ccache: {0} of {1} compilations were cache hits ({2}%); cache size {3:F0} of {4:F0} MiB after cleanup", + $hits, $total, $pct, ($s['cache_size_kibibyte'] / 1024), ($s['max_cache_size_kibibyte'] / 1024)) + echo $msg + Add-AppveyorMessage $msg +} catch { + # Cleanup and statistics are best-effort, do not fail the build over them + echo "SKIP: Could not report ccache statistics: $_" + $global:LASTEXITCODE = 0 +} diff --git a/NEWS.adoc b/NEWS.adoc index 67077f8612..5742af7293 100644 --- a/NEWS.adoc +++ b/NEWS.adoc @@ -1041,6 +1041,15 @@ https://github.com/networkupstools/nut/milestone/13 * Further rearrangement of AppVeyor CI builds and security hardening of GitHub Actions used to prepare the dist tarballs and report on progress there. [PRs #3753, #3754, #3756] + * Fixed more AppVeyor CI step scripts where `cmd.exe` interpreted `||` + inside the single-quoted `bash` command lines itself (it does not + honour such quotes). Also enforce the `ccache` size limit before the + build cache is saved (it could outgrow the limit for a saved cache + item, and get lost), do not save the build cache from pull request + builds (they still restore it) nor from failed builds other than of + `master`, run the tests in parallel, sped up compression of the + artifact archive, and report per-build `ccache` statistics in the + GitHub notification of passed builds. * CI build matrix scenarios (Jenkins autotools builds, `ci_build.sh` with `BUILD_TYPE=default-withdoc*`) now pass `--disable-spellcheck`, so their `make check` no longer runs (and fails on) the documentation spellcheck. diff --git a/appveyor.yml b/appveyor.yml index 8ae93250fa..8c2e2211c5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,7 +27,8 @@ platform: x64 # https://www.appveyor.com/docs/build-cache/ environment: - APPVEYOR_SAVE_CACHE_ON_ERROR: true + # Note: APPVEYOR_SAVE_CACHE_ON_ERROR and APPVEYOR_CACHE_SKIP_SAVE are set + # in the init section below, depending on the branch and PR status. # Store cache entries without (re-)compression: the data is mostly already # compressed (ccache objects, pacman *.zst packages), and single-threaded # LZMA at -mx=6 took 1-2 minutes per run for the ccache entry alone. @@ -39,6 +40,7 @@ environment: # (see ci_build.sh for details): CI_AUTOGEN_STABLE_VERSION: true CCACHE_DIR: /home/appveyor/.ccache + CCACHE_MAXSIZE: 750MiB DO_CLEAN_NUTCI_CACHE: no DO_USE_NUTCI_CACHE: yes MSYS: "winsymlinks:nativestrict" @@ -78,6 +80,20 @@ notifications: init: - ps: | Set-AppveyorBuildVariable -Name CI_DEADLINE_EPOCH -Value ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + 51 * 60) + # There is one build cache per project, shared by all branches and PRs, + # and each save replaces the previous entries. So PR builds only restore + # the cache (warmed up by master builds), but do not save it: otherwise + # each PR build pushes objects of master (and other PRs) out of the + # limited cache space, and spends time to upload it all again. This is + # only a convenience, since a PR can change this file: to really keep + # PR builds from altering the cache used by master builds, untick "Save + # build cache in Pull Request builds" in the AppVeyor project settings. + # Builds of master also save the cache when they fail (e.g. by the + # deadline above), so that a cold cache gets warm over several builds; + # other branch builds only save it when they succeed. Note that for PR + # builds APPVEYOR_REPO_BRANCH is the target branch. + - ps: | + if ($env:APPVEYOR_PULL_REQUEST_NUMBER) { $env:APPVEYOR_CACHE_SKIP_SAVE = "true" } elseif ($env:APPVEYOR_REPO_BRANCH -eq "master") { $env:APPVEYOR_SAVE_CACHE_ON_ERROR = "true" } # The steps below are kept in scripts named after each section, under # .github/scripts/appveyor/ (.bat for `cmd`, .ps1 for `ps` steps), @@ -87,6 +103,12 @@ init: # 08-after_test, ...), see https://www.appveyor.com/docs/build-configuration/#build-pipeline # Note that `init` above runs before the repository is cloned, so it # has to stay inline. +# NOTE for the .bat scripts: cmd.exe only treats double quotes as quoting, +# not the single quotes around `bash -lc '...'` arguments, so any `|`, `||`, +# `&`, `&&`, `<` or `>` outside of double quotes there is interpreted by +# cmd itself (e.g. `a || b` runs `b` only as a cmd command if bash failed, +# and silently drops everything after it otherwise). Use `if ! a ; then +# b ; fi` style constructs (or double-quote such characters) instead. install: - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\02-install.bat" @@ -106,6 +128,11 @@ after_test: - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\08-after_test.bat" - ps: '& "$env:APPVEYOR_BUILD_FOLDER\.github\scripts\appveyor\08-after_test.ps1"' +# Note: AppVeyor saves the build cache after on_success scripts (and for +# failed master builds right after the failing step, before on_failure), see +# https://www.appveyor.com/docs/build-configuration/#build-pipeline +on_success: + - ps: '& "$env:APPVEYOR_BUILD_FOLDER\.github\scripts\appveyor\12-on_success.ps1"' artifacts: - path: 'NUT-for-Windows*.7z'