AppVeyor: fix cmd.exe-eaten "||", keep the ccache cache from getting lost, no cache saves from PRs

.github/scripts/appveyor/02-install.bat

@@ -45,5 +45,5 @@ REM Preserve the current working directory:
 set CHERE_INVOKING=yes
 REM Start a 64 bit Mingw environment:
 set MSYSTEM=MINGW64
-C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; ccache -sv || echo "SKIP: Could not query ccache stats" ; ccache -o sloppiness=file_macro || true ; ccache -o compression=true || true ; ccache -o max_size=750M || true '
+C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; if ! ccache -sv ; then echo "SKIP: Could not query ccache stats" ; fi ; ccache -o compression=true ; echo "ccache settings: max_size=`ccache -k max_size` compression=`ccache -k compression`" ; true '
 if errorlevel 1 exit /b %ERRORLEVEL%

.github/scripts/appveyor/03-before_build.bat

@@ -19,5 +19,9 @@ REM Preserve the current working directory:
 set CHERE_INVOKING=yes
 REM Start a 64 bit Mingw environment:
 set MSYSTEM=MINGW64
+REM Zero the ccache statistics (02-install.bat printed the restored ones),
+REM so that reports of later steps (05, 08, 12) count just this build:
+C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -z ; true '
+if errorlevel 1 exit /b %ERRORLEVEL%
 C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; export MSYS2_PATH ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" bash ./scripts/Windows/build-mingw-prereqs.sh'
 if errorlevel 1 exit /b %ERRORLEVEL%

.github/scripts/appveyor/04-build_script.bat

@@ -21,5 +21,9 @@ REM to find "nearby" program or configuration files (see common.c
 REM for current implementation). Hard-coded fallback strings may
 REM end up getting used in those cases.
 REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml
-C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss timeout -k 30 "$R" ./ci_build.sh --with-docs=no'
+REM The "ccache -c" runs whatever the build result: AppVeyor saves the cache of
+REM a failed (or deadline-terminated) build right after the failing step, so
+REM this enforces ccache max_size before that. Successful builds also get it in
+REM 12-on_success.ps1 (after the later steps which compile tests and bundles).
+C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss timeout -k 30 "$R" ./ci_build.sh --with-docs=no ; RES=$? ; if ! PATH="/mingw64/bin:$PATH" ccache -c ; then echo "SKIP: Could not clean up ccache" ; fi ; exit "$RES" '
 if errorlevel 1 exit /b %ERRORLEVEL%

.github/scripts/appveyor/05-after_build.bat

@@ -9,5 +9,5 @@ REM Preserve the current working directory:
 set CHERE_INVOKING=yes
 REM Start a 64 bit Mingw environment:
 set MSYSTEM=MINGW64
-C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"'
+C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; if ! ccache -sv ; then if ! ccache -s ; then echo "SKIP: Could not query ccache stats" ; fi ; fi'
 if errorlevel 1 exit /b %ERRORLEVEL%

.github/scripts/appveyor/07-test_script.bat

@@ -10,8 +10,9 @@ set CHERE_INVOKING=yes
 REM Start a 64 bit Mingw environment:
 set MSYSTEM=MINGW64
 REM Time limits: see CI_DEADLINE_EPOCH in appveyor.yml
-REM Start Mingw-based integration and unit checks:
-C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s check || bash -lc "for F in tests/*.log tests/*.trs ; do echo \"===---=== $F :\"; cat \"$F\"; done; exit 1;" '
+REM Start Mingw-based integration and unit checks (in parallel: measured locally,
+REM 16 tests passed in all of 7 runs, with -j2 taking ~30% less time than serial):
+C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 2 check ; RES=$? ; if [ "$RES" -ne 0 ] ; then for F in tests/*.log tests/*.trs ; do echo "===---=== $F :" ; cat "$F" ; done ; exit "$RES" ; fi '
 if errorlevel 1 exit /b %ERRORLEVEL%
 REM Start a Mingw-based documentation spellcheck (note that "make check" above could cover some documents in path of artifact delivery, but maybe not all that we have):
 C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 spellcheck'

.github/scripts/appveyor/08-after_test.bat

@@ -14,19 +14,20 @@ REM Note: currently Python installation path with MSYS is buggy [#1584]
 REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml
 C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; set -e ; if ! rm -rf ".inst" ; then echo "WARNING: Failed to clean away .inst" ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 install-win-bundle DESTDIR="`pwd`/.inst/NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%"'
 if errorlevel 1 exit /b %ERRORLEVEL%
-C:\msys64\usr\bin\bash -lc 'date -u; set -e ; rm -rf ./.inst/NUT-for-Windows-x86_64-SNAPSHOT || true ; ln -fs "NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%" ./.inst/NUT-for-Windows-x86_64-SNAPSHOT ; ( cd .inst/NUT-for-Windows-x86_64-SNAPSHOT ; find . -ls ; )'
-if errorlevel 1 exit /b %ERRORLEVEL%
 C:\msys64\usr\bin\bash -lc 'date -u'
 pushd .inst
 if errorlevel 1 exit /b %ERRORLEVEL%
-REM 7z -snh: hoping for a version that can save hardlinks as such, and in 7z format to boot:
-7z a -snh ../NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%.7z NUT*
+REM 7z -snh: hoping for a version that can save hardlinks as such, and in 7z format to boot.
+REM -mx=3 -mmt=on: the default level took about 4 minutes of the CI time limit
+REM for this (debug build) archive alone. Measured on a 679 MB bundle with two
+REM threads: default 98s/52MB, -mx=3 10s/76MB, -mx=1 12s/199MB (other method).
+7z a -snh -mx=3 -mmt=on ../NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%.7z NUT*
 if errorlevel 1 exit /b %ERRORLEVEL%
 popd
 
 REM Report ccache stats (here rather than in a separate yml step after the
 REM artifact message in 08-after_test.ps1, the order does not matter):
-C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"'
+C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; if ! ccache -sv ; then if ! ccache -s ; then echo "SKIP: Could not query ccache stats" ; fi ; fi'
 if errorlevel 1 exit /b %ERRORLEVEL%
-C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -x || echo "SKIP: Could not query ccache compression stats"'
+C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; if ! ccache -x ; then echo "SKIP: Could not query ccache compression stats" ; fi'
 if errorlevel 1 exit /b %ERRORLEVEL%

.github/scripts/appveyor/12-on_success.ps1

@@ -0,0 +1,42 @@
+# Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows
+# with MSYS2/MinGW x64: enforce ccache max_size before the cache save, and
+# report ccache statistics of this build (counters are zeroed in
+# 03-before_build.bat) in the job messages, which the GitHub notification
+# template in appveyor.yml includes for passed builds.
+
+# Run ccache through MSYS2 bash: CCACHE_DIR from appveyor.yml is a POSIX path,
+# which the MSYS2 runtime converts for ccache.exe like during the build itself.
+# Keep the bash command free of double quotes (Windows PowerShell 5.1 mangles
+# them in native command arguments), and print everything to stdout.
+$env:CHERE_INVOKING = "yes"
+$env:MSYSTEM = "MINGW64"
+
+try {
+    # AppVeyor saves the build cache right after the on_success scripts, so
+    # clean up first (later steps also compiled tests and bundles after the
+    # cleanup at the end of 04-build_script.bat); then the reported cache size
+    # is what gets saved.
+    $out = & C:\msys64\usr\bin\bash -lc 'date -u ; PATH=/mingw64/bin:$PATH ; export PATH ; if ! ccache -c ; then echo SKIP: Could not clean up ccache ; fi ; ccache --print-stats'
+    if ($LASTEXITCODE -ne 0) { throw "ccache --print-stats exited with code $LASTEXITCODE" }
+    $s = @{}
+    $out | ForEach-Object {
+        $kv = $_ -split "`t", 2
+        if ($kv.Count -eq 2) { $s[$kv[0]] = [long]$kv[1] } else { echo $_ }
+    }
+    if ($s.Count -eq 0) { throw "no ccache statistics" }
+
+    $hits = $s['direct_cache_hit'] + $s['preprocessed_cache_hit']
+    $total = $hits + $s['cache_miss']
+    $pct = 0
+    if ($total -gt 0) { $pct = [math]::Round(100.0 * $hits / $total, 1) }
+    # Invariant culture: same number formatting whatever the worker locale is
+    $msg = [string]::Format([System.Globalization.CultureInfo]::InvariantCulture,
+        "ccache: {0} of {1} compilations were cache hits ({2}%); cache size {3:F0} of {4:F0} MiB after cleanup",
+        $hits, $total, $pct, ($s['cache_size_kibibyte'] / 1024), ($s['max_cache_size_kibibyte'] / 1024))
+    echo $msg
+    Add-AppveyorMessage $msg
+} catch {
+    # Cleanup and statistics are best-effort, do not fail the build over them
+    echo "SKIP: Could not report ccache statistics: $_"
+    $global:LASTEXITCODE = 0
+}

NEWS.adoc

@@ -1041,6 +1041,15 @@ https://github.com/networkupstools/nut/milestone/13
     * Further rearrangement of AppVeyor CI builds and security hardening of
       GitHub Actions used to prepare the dist tarballs and report on progress
       there. [PRs #3753, #3754, #3756]
+    * Fixed more AppVeyor CI step scripts where `cmd.exe` interpreted `||`
+      inside the single-quoted `bash` command lines itself (it does not
+      honour such quotes). Also enforce the `ccache` size limit before the
+      build cache is saved (it could outgrow the limit for a saved cache
+      item, and get lost), do not save the build cache from pull request
+      builds (they still restore it) nor from failed builds other than of
+      `master`, run the tests in parallel, sped up compression of the
+      artifact archive, and report per-build `ccache` statistics in the
+      GitHub notification of passed builds.
     * CI build matrix scenarios (Jenkins autotools builds, `ci_build.sh` with
       `BUILD_TYPE=default-withdoc*`) now pass `--disable-spellcheck`, so their
       `make check` no longer runs (and fails on) the documentation spellcheck.

appveyor.yml

@@ -27,7 +27,8 @@ platform: x64
 
 # https://www.appveyor.com/docs/build-cache/
 environment:
-  APPVEYOR_SAVE_CACHE_ON_ERROR: true
+  # Note: APPVEYOR_SAVE_CACHE_ON_ERROR and APPVEYOR_CACHE_SKIP_SAVE are set
+  # in the init section below, depending on the branch and PR status.
   # Store cache entries without (re-)compression: the data is mostly already
   # compressed (ccache objects, pacman *.zst packages), and single-threaded
   # LZMA at -mx=6 took 1-2 minutes per run for the ccache entry alone.
@@ -39,6 +40,7 @@ environment:
   # (see ci_build.sh for details):
   CI_AUTOGEN_STABLE_VERSION: true
   CCACHE_DIR: /home/appveyor/.ccache
+  CCACHE_MAXSIZE: 750MiB
   DO_CLEAN_NUTCI_CACHE: no
   DO_USE_NUTCI_CACHE: yes
   MSYS: "winsymlinks:nativestrict"
@@ -78,6 +80,20 @@ notifications:
 init:
   - ps: |
         Set-AppveyorBuildVariable -Name CI_DEADLINE_EPOCH -Value ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + 51 * 60)
+  # There is one build cache per project, shared by all branches and PRs,
+  # and each save replaces the previous entries. So PR builds only restore
+  # the cache (warmed up by master builds), but do not save it: otherwise
+  # each PR build pushes objects of master (and other PRs) out of the
+  # limited cache space, and spends time to upload it all again. This is
+  # only a convenience, since a PR can change this file: to really keep
+  # PR builds from altering the cache used by master builds, untick "Save
+  # build cache in Pull Request builds" in the AppVeyor project settings.
+  # Builds of master also save the cache when they fail (e.g. by the
+  # deadline above), so that a cold cache gets warm over several builds;
+  # other branch builds only save it when they succeed. Note that for PR
+  # builds APPVEYOR_REPO_BRANCH is the target branch.
+  - ps: |
+        if ($env:APPVEYOR_PULL_REQUEST_NUMBER) { $env:APPVEYOR_CACHE_SKIP_SAVE = "true" } elseif ($env:APPVEYOR_REPO_BRANCH -eq "master") { $env:APPVEYOR_SAVE_CACHE_ON_ERROR = "true" }
 
 # The steps below are kept in scripts named after each section, under
 # .github/scripts/appveyor/ (.bat for `cmd`, .ps1 for `ps` steps),
@@ -87,6 +103,12 @@ init:
 # 08-after_test, ...), see https://www.appveyor.com/docs/build-configuration/#build-pipeline
 # Note that `init` above runs before the repository is cloned, so it
 # has to stay inline.
+# NOTE for the .bat scripts: cmd.exe only treats double quotes as quoting,
+# not the single quotes around `bash -lc '...'` arguments, so any `|`, `||`,
+# `&`, `&&`, `<` or `>` outside of double quotes there is interpreted by
+# cmd itself (e.g. `a || b` runs `b` only as a cmd command if bash failed,
+# and silently drops everything after it otherwise). Use `if ! a ; then
+# b ; fi` style constructs (or double-quote such characters) instead.
 install:
   - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\02-install.bat"
 
@@ -106,6 +128,11 @@ after_test:
   - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\08-after_test.bat"
   - ps: '& "$env:APPVEYOR_BUILD_FOLDER\.github\scripts\appveyor\08-after_test.ps1"'
 
+# Note: AppVeyor saves the build cache after on_success scripts (and for
+# failed master builds right after the failing step, before on_failure), see
+# https://www.appveyor.com/docs/build-configuration/#build-pipeline
+on_success:
+  - ps: '& "$env:APPVEYOR_BUILD_FOLDER\.github\scripts\appveyor\12-on_success.ps1"'
 
 artifacts:
   - path: 'NUT-for-Windows*.7z'