diff --git a/.github/scripts/appveyor/02-install.bat b/.github/scripts/appveyor/02-install.bat new file mode 100644 index 0000000000..6914e4715a --- /dev/null +++ b/.github/scripts/appveyor/02-install.bat @@ -0,0 +1,49 @@ +REM Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +REM with MSYS2/MinGW x64. Called by AppVeyor with the build folder as current +REM directory. AppVeyor checks each line of inline yml scripts, but only the +REM exit code of a called .bat file, so we check each command here. +REM +REM Copyright (C) 2022-2026 Jim Klimov + +REM Do not give pacman cause for complaints: +C:\msys64\usr\bin\bash -lc "date -u; mkdir -p /var/cache/pacman/pkg; ls -la /" +if errorlevel 1 exit /b %ERRORLEVEL% + +REM FIXME: Since Jan 2025, normal update below fails with the message: +REM installing mingw-w64-i686-gdb (16.1-1) breaks dependency +REM 'mingw-w64-i686-gdb=15.2' required by mingw-w64-i686-gdb-multiarch +REM Currently we sweep it away by removing the conflicting package: +REM C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -R mingw-w64-i686-gdb-multiarch || true" + +REM FIXME: Make this update or not of the ecosystem conditional? +REM Store a copy of C:\msys64 elsewhere as a tarball? +REM As of Aug 2025, the first update is quick (15sec, 18Mb DL, 91MB installed), +REM the second is costlier (6.5min, 378MB DL, 2.5GB installed) +REM Core update (in case any core packages are outdated): +REM C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -Syuu" +REM Normal update (same command again): +REM C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -Syuu" + +REM As of Aug 2025, with updates commented away above still in place +REM so many packages skipped, this step did cost additional 58MB DL, +REM 392MB installed (new data), 1min of work. +REM Without the ecosystem update above, the overhead of this step remained +REM similar. It did however use a smaller population (via deps) and older +REM versions of packages. +REM Prerequisites for NUT per https://github.com/networkupstools/nut/blob/master/docs/config-prereqs.txt : +C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -S --needed base-devel mingw-w64-x86_64-toolchain autoconf-wrapper automake-wrapper libtool mingw-w64-x86_64-libltdl gcc ccache mingw-w64-x86_64-ccache git aspell aspell-en python mingw-w64-x86_64-python-pygments mingw-w64-x86_64-winpthreads-git mingw-w64-x86_64-libusb mingw-w64-x86_64-libusb-compat-git mingw-w64-x86_64-neon libneon-devel mingw-w64-x86_64-libgd mingw-w64-x86_64-cppunit mingw-w64-x86_64-nss mingw-w64-x86_64-openssl" +if errorlevel 1 exit /b %ERRORLEVEL% +REM SKIP mingw-w64-x86_64-libmodbus-git : we custom-build one with USB support +REM SKIP for now NUT-Monitor prereqs (runtime Python would require somilar modules; need to fix localization builds like "fr.po"): gettext mingw-w64-x86_64-python-pyqt6 + +REM FIXME: Make these stats conditional for just troubleshooting sessions? +REM It seems the first "du" call costs us about 1-2 minutes (different runs): +REM Assorted stats after package processing: +REM C:\msys64\usr\bin\bash -lc "date -u; ls -la / ; du -ksx / ; date -u; du -ks /var/cache/pacman/pkg; date -u" + +REM Preserve the current working directory: +set CHERE_INVOKING=yes +REM Start a 64 bit Mingw environment: +set MSYSTEM=MINGW64 +C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; ccache -sv || echo "SKIP: Could not query ccache stats" ; ccache -o sloppiness=file_macro || true ; ccache -o compression=true || true ; ccache -o max_size=750M || true ' +if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/03-before_build.bat b/.github/scripts/appveyor/03-before_build.bat new file mode 100644 index 0000000000..eaad237eba --- /dev/null +++ b/.github/scripts/appveyor/03-before_build.bat @@ -0,0 +1,23 @@ +REM Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +REM with MSYS2/MinGW x64. Called by AppVeyor with the build folder as current +REM directory. AppVeyor checks each line of inline yml scripts, but only the +REM exit code of a called .bat file, so we check each command here. +REM +REM Copyright (C) 2022-2026 Jim Klimov + +REM NOTE: The script aims to configure for minimal build results, e.g. +REM no tools and docs, and to ensure parallel builds via MAKEFLAGS... +REM As of Aug 2025, in cases these artifacts (and ~/.ccache) were not cached +REM and the step did run, it cost us 22min (before disabling tools and docs). + +REM Ensure we have a net-snmp to build against +REM Adapted from scripts/Windows/README.adoc document. +REM Here we hope to build it once, then use the +REM stashed version across Appveyor rebuilds. +REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml +REM Preserve the current working directory: +set CHERE_INVOKING=yes +REM Start a 64 bit Mingw environment: +set MSYSTEM=MINGW64 +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; export MSYS2_PATH ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" bash ./scripts/Windows/build-mingw-prereqs.sh' +if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/04-build_script.bat b/.github/scripts/appveyor/04-build_script.bat new file mode 100644 index 0000000000..ac2442246e --- /dev/null +++ b/.github/scripts/appveyor/04-build_script.bat @@ -0,0 +1,25 @@ +REM Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +REM with MSYS2/MinGW x64. Called by AppVeyor with the build folder as current +REM directory. AppVeyor checks each line of inline yml scripts, but only the +REM exit code of a called .bat file, so we check each command here. +REM +REM Copyright (C) 2022-2026 Jim Klimov + +REM Preserve the current working directory: +set CHERE_INVOKING=yes +REM Start a 64 bit Mingw environment: +set MSYSTEM=MINGW64 +REM Note: currently we save job time and do not install asciidoc/a2x +REM # --with-docs="man=auto html-single=auto html-chunked=no pdf=no" +REM The resulting configuration defaults to --prefix=/mingw{32,64} +REM as appropriate for the platform, which ends up as a subdirectory +REM under DESTDIR when we `make install` later, and in our 7z archive. +REM The builds are relocatable however, so end-users can extract to +REM e.g. "C:\Program Files\NUT" and forfeit "mingwXX" part, probably, +REM but this can complicate automatic relative directory resolution +REM to find "nearby" program or configuration files (see common.c +REM for current implementation). Hard-coded fallback strings may +REM end up getting used in those cases. +REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss timeout -k 30 "$R" ./ci_build.sh --with-docs=no' +if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/05-after_build.bat b/.github/scripts/appveyor/05-after_build.bat new file mode 100644 index 0000000000..3ebf4e15a8 --- /dev/null +++ b/.github/scripts/appveyor/05-after_build.bat @@ -0,0 +1,13 @@ +REM Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +REM with MSYS2/MinGW x64. Called by AppVeyor with the build folder as current +REM directory. AppVeyor checks each line of inline yml scripts, but only the +REM exit code of a called .bat file, so we check each command here. +REM +REM Copyright (C) 2022-2026 Jim Klimov + +REM Preserve the current working directory: +set CHERE_INVOKING=yes +REM Start a 64 bit Mingw environment: +set MSYSTEM=MINGW64 +C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"' +if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/07-test_script.bat b/.github/scripts/appveyor/07-test_script.bat new file mode 100644 index 0000000000..aa1a9179c6 --- /dev/null +++ b/.github/scripts/appveyor/07-test_script.bat @@ -0,0 +1,18 @@ +REM Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +REM with MSYS2/MinGW x64. Called by AppVeyor with the build folder as current +REM directory. AppVeyor checks each line of inline yml scripts, but only the +REM exit code of a called .bat file, so we check each command here. +REM +REM Copyright (C) 2022-2026 Jim Klimov + +REM Preserve the current working directory: +set CHERE_INVOKING=yes +REM Start a 64 bit Mingw environment: +set MSYSTEM=MINGW64 +REM Time limits: see CI_DEADLINE_EPOCH in appveyor.yml +REM Start Mingw-based integration and unit checks: +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s check || bash -lc "for F in tests/*.log tests/*.trs ; do echo \"===---=== $F :\"; cat \"$F\"; done; exit 1;" ' +if errorlevel 1 exit /b %ERRORLEVEL% +REM Start a Mingw-based documentation spellcheck (note that "make check" above could cover some documents in path of artifact delivery, but maybe not all that we have): +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 spellcheck' +if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/08-after_test.bat b/.github/scripts/appveyor/08-after_test.bat new file mode 100644 index 0000000000..a0195f2af0 --- /dev/null +++ b/.github/scripts/appveyor/08-after_test.bat @@ -0,0 +1,32 @@ +REM Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +REM with MSYS2/MinGW x64. Called by AppVeyor with the build folder as current +REM directory. AppVeyor checks each line of inline yml scripts, but only the +REM exit code of a called .bat file, so we check each command here. +REM +REM Copyright (C) 2022-2026 Jim Klimov + +REM Preserve the current working directory: +set CHERE_INVOKING=yes +REM Start a 64 bit Mingw environment: +set MSYSTEM=MINGW64 +REM Oh the joys of shell scripting with strings passed through CMD: +REM Note: currently Python installation path with MSYS is buggy [#1584] +REM Time limit: see CI_DEADLINE_EPOCH in appveyor.yml +C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; set -e ; if ! rm -rf ".inst" ; then echo "WARNING: Failed to clean away .inst" ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 install-win-bundle DESTDIR="`pwd`/.inst/NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%"' +if errorlevel 1 exit /b %ERRORLEVEL% +C:\msys64\usr\bin\bash -lc 'date -u; set -e ; rm -rf ./.inst/NUT-for-Windows-x86_64-SNAPSHOT || true ; ln -fs "NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%" ./.inst/NUT-for-Windows-x86_64-SNAPSHOT ; ( cd .inst/NUT-for-Windows-x86_64-SNAPSHOT ; find . -ls ; )' +if errorlevel 1 exit /b %ERRORLEVEL% +C:\msys64\usr\bin\bash -lc 'date -u' +pushd .inst +if errorlevel 1 exit /b %ERRORLEVEL% +REM 7z -snh: hoping for a version that can save hardlinks as such, and in 7z format to boot: +7z a -snh ../NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%.7z NUT* +if errorlevel 1 exit /b %ERRORLEVEL% +popd + +REM Report ccache stats (here rather than in a separate yml step after the +REM artifact message in 08-after_test.ps1, the order does not matter): +C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"' +if errorlevel 1 exit /b %ERRORLEVEL% +C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -x || echo "SKIP: Could not query ccache compression stats"' +if errorlevel 1 exit /b %ERRORLEVEL% diff --git a/.github/scripts/appveyor/08-after_test.ps1 b/.github/scripts/appveyor/08-after_test.ps1 new file mode 100644 index 0000000000..e1465cada6 --- /dev/null +++ b/.github/scripts/appveyor/08-after_test.ps1 @@ -0,0 +1,11 @@ +# Network UPS Tools: AppVeyor CI recipe step (see appveyor.yml), NUT for Windows +# with MSYS2/MinGW x64: report a link to the built archive in the job messages. +# +# Copyright (C) 2022-2026 Jim Klimov + +# Consider APPVEYOR_PULL_REQUEST_HEAD_COMMIT (PR) vs APPVEYOR_REPO_COMMIT (target branch, or ephemeral) +$nut7z = "NUT-for-Windows-x86_64-SNAPSHOT-$($env:APPVEYOR_BUILD_VERSION).7z" +$msg = "A 7-zip archive with built NUT for Windows debug binaries is temporarily available: [$($nut7z)]($($env:APPVEYOR_URL)/api/buildjobs/$($env:APPVEYOR_JOB_ID)/artifacts/$($nut7z))" +echo $msg +echo "build-id:$($env:APPVEYOR_BUILD_ID) job-id:$($env:APPVEYOR_JOB_ID) prj-id:$($env:APPVEYOR_PROJECT_ID)" +Add-AppveyorMessage $msg diff --git a/appveyor.yml b/appveyor.yml index 3a77e286be..8ae93250fa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,16 @@ platform: x64 # https://www.appveyor.com/docs/build-cache/ environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true - APPVEYOR_CACHE_ENTRY_ZIP_ARGS: -t7z -m0=lzma -mx=6 + # Store cache entries without (re-)compression: the data is mostly already + # compressed (ccache objects, pacman *.zst packages), and single-threaded + # LZMA at -mx=6 took 1-2 minutes per run for the ccache entry alone. + # The cache limit for PR builds is 1GB total across all entries, so + # ccache max_size is capped below (see install steps) to leave a margin + # for the other entries (pacman packages ~100MB, nut-win-deps, nut-ci): + APPVEYOR_CACHE_ENTRY_ZIP_ARGS: -t7z -mx=0 + # Keep config.h stable across PRs, so ccache can reuse -g3 objects + # (see ci_build.sh for details): + CI_AUTOGEN_STABLE_VERSION: true CCACHE_DIR: /home/appveyor/.ccache DO_CLEAN_NUTCI_CACHE: no DO_USE_NUTCI_CACHE: yes @@ -57,143 +66,45 @@ notifications: # or look for the chapter in nearby lines in later (current) revisions. # Note: not using `time` in scripts currently - they did upset # AppVeyor console log scanner with a /^sys.*/ match (apparently) +# AppVeyor kills the job at 60 minutes, without saving the build cache: +# a cold-ccache build then never gets warm, and an interrupted cache save +# can even lose the old entry. So we set a deadline at 51 minutes after +# the job start, and our build/test/bundle steps fail (via `timeout`) if +# they would run past it. With APPVEYOR_SAVE_CACHE_ON_ERROR this leaves +# a few minutes for the 7z step, artifact upload and saving the cache. +# Each bash step below computes its time left like this (no CMD special +# characters, so it passes through CMD intact): +# R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) +init: + - ps: | + Set-AppveyorBuildVariable -Name CI_DEADLINE_EPOCH -Value ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + 51 * 60) + +# The steps below are kept in scripts named after each section, under +# .github/scripts/appveyor/ (.bat for `cmd`, .ps1 for `ps` steps), +# numbered by the position of their section among the AppVeyor +# pipeline script phases (01-init, 02-install, 03-before_build, +# 04-build_script, 05-after_build, 06-before_test, 07-test_script, +# 08-after_test, ...), see https://www.appveyor.com/docs/build-configuration/#build-pipeline +# Note that `init` above runs before the repository is cloned, so it +# has to stay inline. install: - - cmd: | - REM Do not give pacman cause for complaints: - C:\msys64\usr\bin\bash -lc "date -u; mkdir -p /var/cache/pacman/pkg; ls -la /" - -# - cmd: | -# REM FIXME: Since Jan 2025, normal update below fails with the message: -# REM installing mingw-w64-i686-gdb (16.1-1) breaks dependency -# REM 'mingw-w64-i686-gdb=15.2' required by mingw-w64-i686-gdb-multiarch -# REM Currently we sweep it away by removing the conflicting package: -# C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -R mingw-w64-i686-gdb-multiarch || true" - -# FIXME: Make this update or not of the ecosystem conditional? -# Store a copy of C:\msys64 elsewhere as a tarball? -# As of Aug 2025, the first update is quick (15sec, 18Mb DL, 91MB installed), -# the second is costlier (6.5min, 378MB DL, 2.5GB installed) -# - cmd: | -# REM Core update (in case any core packages are outdated): -# C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -Syuu" -# - cmd: | -# REM Normal update (same command again): -# C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -Syuu" - -# As of Aug 2025, with updates commented away above still in place -# so many packages skipped, this step did cost additional 58MB DL, -# 392MB installed (new data), 1min of work. -# Without the ecosystem update above, the overhead of this step remained -# similar. It did however use a smaller population (via deps) and older -# versions of packages. - - cmd: | - REM Prerequisites for NUT per https://github.com/networkupstools/nut/blob/master/docs/config-prereqs.txt : - C:\msys64\usr\bin\bash -lc "date -u; pacman --noconfirm -S --needed base-devel mingw-w64-x86_64-toolchain autoconf-wrapper automake-wrapper libtool mingw-w64-x86_64-libltdl gcc ccache mingw-w64-x86_64-ccache git aspell aspell-en python mingw-w64-x86_64-python-pygments mingw-w64-x86_64-winpthreads-git mingw-w64-x86_64-libusb mingw-w64-x86_64-libusb-compat-git mingw-w64-x86_64-neon libneon-devel mingw-w64-x86_64-libgd mingw-w64-x86_64-cppunit mingw-w64-x86_64-nss mingw-w64-x86_64-openssl" - REM SKIP mingw-w64-x86_64-libmodbus-git : we custom-build one with USB support - REM SKIP for now NUT-Monitor prereqs (runtime Python would require somilar modules; need to fix localization builds like "fr.po"): gettext mingw-w64-x86_64-python-pyqt6 - -# FIXME: Make these stats conditional for just troubleshooting sessions? -# It seems the first "du" call costs us about 1-2 minutes (different runs): -# - cmd: | -# REM Assorted stats after package processing: -# C:\msys64\usr\bin\bash -lc "date -u; ls -la / ; du -ksx / ; date -u; du -ks /var/cache/pacman/pkg; date -u" - - - cmd: | - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; ccache -sv || echo "SKIP: Could not query ccache stats" ; ccache -o sloppiness=file_macro || true ; ccache -o compression=true || true ' - - -# NOTE: The script aims to configure for minimal build results, e.g. -# no tools and docs, and to ensure parallel builds via MAKEFLAGS... -# As of Aug 2025, in cases these artifacts (and ~/.ccache) were not cached -# and the step did run, it cost us 22min (before disabling tools and docs). -before_build: - - cmd: | - REM Ensure we have a net-snmp to build against - REM Adapted from scripts/Windows/README.adoc document. - REM Here we hope to build it once, then use the - REM stashed version across Appveyor rebuilds. - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - C:\msys64\usr\bin\bash -lc 'date -u; export MSYS2_PATH ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" bash ./scripts/Windows/build-mingw-prereqs.sh' + - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\02-install.bat" +before_build: + - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\03-before_build.bat" build_script: - - cmd: | - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - REM Note: currently we save job time and do not install asciidoc/a2x - REM # --with-docs="man=auto html-single=auto html-chunked=no pdf=no" - REM The resulting configuration defaults to --prefix=/mingw{32,64} - REM as appropriate for the platform, which ends up as a subdirectory - REM under DESTDIR when we `make install` later, and in our 7z archive. - REM The builds are relocatable however, so end-users can extract to - REM e.g. "C:\Program Files\NUT" and forfeit "mingwXX" part, probably, - REM but this can complicate automatic relative directory resolution - REM to find "nearby" program or configuration files (see common.c - REM for current implementation). Hard-coded fallback strings may - REM end up getting used in those cases. - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss ./ci_build.sh --with-docs=no' - + - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\04-build_script.bat" after_build: - - cmd: | - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"' - + - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\05-after_build.bat" test_script: - - cmd: | - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - REM Start Mingw-based integration and unit checks: - C:\msys64\usr\bin\bash -lc 'date -u; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" make -s check || bash -lc "for F in tests/*.log tests/*.trs ; do echo \"===---=== $F :\"; cat \"$F\"; done; exit 1;" ' - REM Start a Mingw-based documentation spellcheck (note that "make check" above could cover some documents in path of artifact delivery, but maybe not all that we have): - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" make -s -j 8 spellcheck' - + - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\07-test_script.bat" after_test: - - cmd: | - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - REM Oh the joys of shell scripting with strings passed through CMD: - REM Note: currently Python installation path with MSYS is buggy [#1584] - C:\msys64\usr\bin\bash -lc 'date -u; set -e ; if ! rm -rf ".inst" ; then echo "WARNING: Failed to clean away .inst" ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" make -s -j 8 install-win-bundle DESTDIR="`pwd`/.inst/NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%"' - C:\msys64\usr\bin\bash -lc 'date -u; set -e ; rm -rf ./.inst/NUT-for-Windows-x86_64-SNAPSHOT || true ; ln -fs "NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%" ./.inst/NUT-for-Windows-x86_64-SNAPSHOT ; ( cd .inst/NUT-for-Windows-x86_64-SNAPSHOT ; find . -ls ; )' - C:\msys64\usr\bin\bash -lc 'date -u' - cd .inst - REM 7z -snh: hoping for a version that can save hardlinks as such, and in 7z format to boot: - 7z a -snh ../NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%.7z NUT* - - # Consider APPVEYOR_PULL_REQUEST_HEAD_COMMIT (PR) vs APPVEYOR_REPO_COMMIT (target branch, or ephemeral) - - ps: | - $nut7z = "NUT-for-Windows-x86_64-SNAPSHOT-$($env:APPVEYOR_BUILD_VERSION).7z" - $msg = "A 7-zip archive with built NUT for Windows debug binaries is temporarily available: [$($nut7z)]($($env:APPVEYOR_URL)/api/buildjobs/$($env:APPVEYOR_JOB_ID)/artifacts/$($nut7z))" - echo $msg - echo "build-id:$($env:APPVEYOR_BUILD_ID) job-id:$($env:APPVEYOR_JOB_ID) prj-id:$($env:APPVEYOR_PROJECT_ID)" - Add-AppveyorMessage $msg - - - cmd: | - REM Preserve the current working directory: - set CHERE_INVOKING=yes - REM Start a 64 bit Mingw environment: - set MSYSTEM=MINGW64 - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -sv || ccache -s || echo "SKIP: Could not query ccache stats"' - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" ; export PATH ; ccache -x || echo "SKIP: Could not query ccache compression stats"' + - cmd: call "%APPVEYOR_BUILD_FOLDER%\.github\scripts\appveyor\08-after_test.bat" + - ps: '& "$env:APPVEYOR_BUILD_FOLDER\.github\scripts\appveyor\08-after_test.ps1"' artifacts: diff --git a/ci_build.sh b/ci_build.sh index c781c42540..72efc0e490 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -1135,11 +1135,32 @@ do_autogen_get_CONFIGURE_SCRIPT() { # older system) we have to remove it when we already have the script. # This matches the use-case of distro-building from release tarballs that # include all needed pre-generated files to rely less on OS facilities. - if [ "$CI_OS_NAME" = "windows" ] ; then - $CI_TIME ./autogen.sh || true - else - $CI_TIME ./autogen.sh ### 2>/dev/null - fi || exit + ( + # Optionally drop the PR-specific last component of the version + # (e.g. 2.8.5.1466.13 => 2.8.5.1466) which AC_INIT puts into + # config.h as PACKAGE_VERSION. The ccache direct mode hashes all + # included headers, so a version change misses for nearly every + # file and defeats ccache across pull requests. Only autogen.sh gets + # this setting, so `make` still generates nut_version.h (and the + # UPS_VERSION reported by programs) from actual Git metadata. + if [ x"${CI_AUTOGEN_STABLE_VERSION-}" = xtrue ] && [ -z "${NUT_VERSION_FORCED-}" ] ; then + _NUT_VERSION_STABLE="`NUT_VERSION_QUERY=VER50 ./tools/gitlog2version.sh 2>/dev/null`" || _NUT_VERSION_STABLE="" + case "${_NUT_VERSION_STABLE}" in + *.*.*.*.*) _NUT_VERSION_STABLE="${_NUT_VERSION_STABLE%.*}" ;; + esac + if [ -n "${_NUT_VERSION_STABLE}" ] ; then + echo "INFO: CI_AUTOGEN_STABLE_VERSION: using NUT_VERSION_FORCED='${_NUT_VERSION_STABLE}' for autogen.sh" >&2 + NUT_VERSION_FORCED="${_NUT_VERSION_STABLE}" + export NUT_VERSION_FORCED + fi + fi + + if [ "$CI_OS_NAME" = "windows" ] ; then + $CI_TIME ./autogen.sh || true + else + $CI_TIME ./autogen.sh ### 2>/dev/null + fi + ) || exit fi # Retain the full path to configure script file diff --git a/docs/qa-guide.adoc b/docs/qa-guide.adoc index 0a7e767a22..4227e04f13 100644 --- a/docs/qa-guide.adoc +++ b/docs/qa-guide.adoc @@ -475,7 +475,8 @@ with binary installation prototype area, intended for enthusiastic testing soon after NUT 2.8.0 release. This relies on a few prerequisite packages and a common NUT configuration, -as coded in the `appveyor.yml` file in the NUT codebase. +as coded in the `appveyor.yml` file in the NUT codebase, and the step +scripts it calls from the `.github/scripts/appveyor/` directory. CircleCI ~~~~~~~~ diff --git a/scripts/Windows/dllldd.sh b/scripts/Windows/dllldd.sh index 2ef3dcfe2f..88b056bef8 100755 --- a/scripts/Windows/dllldd.sh +++ b/scripts/Windows/dllldd.sh @@ -81,8 +81,23 @@ discover_COMPILER_PATHS() { } discover_COMPILER_PATHS +# Only scan initialized data sections for DLL names (as older binutils did +# by default), not the whole file: with debug info (-g3) most of our binary +# size is debug sections, so a full scan produced ~200 times more text (all +# piped through filters below), with the same DLL names found. +# Fall back to a whole-file scan if this `strings` lacks the option. +STRINGS_DATA_ONLY="" +if (strings --help 2>&1 | ${GREP} -- '--data' >/dev/null) ; then + STRINGS_DATA_ONLY="-d" +fi + filter_away_system_DLLs() { - ${EGREP} -v -i '^(/.*/)?(msvcrt|userenv|bcrypt|bcryptprimitives|dnsapi|dwrite|iphlpapi|kernelbase|mswsock|shlwapi|winmm|rpcrt4|usp10|ntdll|api-ms-win-[^ ]*|(advapi|crypt|kernel|user|wsock|ws2_|gdi|ole|shell)(32|64))'"${DLLEXT_REGEX_EOL}" + ${EGREP} -v -i '^(/.*/)?(msvcrt|userenv|bcrypt|bcryptprimitives|combase|dnsapi|dwrite|hid|iphlpapi|kernelbase|mswsock|setupapi|shfolder|shlwapi|winmm|wship6|xmllite|rpcrt4|usp10|ntdll|api-ms-win-[^ ]*|(advapi|crypt|kernel|user|wsock|ws2_|gdi|ole|shell)(32|64))'"${DLLEXT_REGEX_EOL}" +} + +first_file_named() { + # Print the first line of stdin (a file path) whose base name is "$1" + awk -v f="$1" '{ n = $0; sub(/^.*\//, "", n); if (n == f) { print; exit } }' } filter_away_NUT_DLLs() { @@ -132,24 +147,37 @@ dllldd_with_tools() ( SEEN=0 NOTSEEN_OD="" if [ -n "${ARCH-}${MINGW_PREFIX-}${MSYSTEM_PREFIX-}" ] ; then + # List candidate locations once, rather than calling `find` for + # each DLL name: spawning processes and walking big directories + # (e.g. /mingw64/bin) is slow on MSYS2, and this adds up quickly + # in recursive searches. Listing order is kept as it was in the + # per-name searches, so the first hit is the same file as before. + DESTDIR_FILES="" + if [ -n "$DESTDIR" -a -d "${DESTDIR}" ] ; then + DESTDIR_FILES="`find \"$DESTDIR\" -type f \! -size 0 2>/dev/null`" + fi + SEARCH_INPUT_FILES="" + if [ -n "$SEARCH_INPUT_PATH" ] ; then + SEARCH_INPUT_FILES="`for D in ${SEARCH_INPUT_PATH_MULTILINE} ; do find \"$D\" -type f \! -size 0 2>/dev/null ; done`" + fi + for OD in objdump "$ARCH-objdump" ; do (command -v "$OD" >/dev/null 2>/dev/null) || continue - ODOUT="`$OD -x \"$@\" 2>/dev/null | ${EGREP} -i '(DLL Name:|^'"${REGEX_WS}"'*NEEDED)' | awk '{print $NF}' | sort | uniq | filter_away_system_DLLs`" \ + # Private headers (-p) hold the PE import table ("DLL Name:") + # and ELF dynamic section ("NEEDED"); -x would also dump the + # (large, for debug builds) symbol tables and relocations. + ODOUT="`$OD -p \"$@\" 2>/dev/null | ${EGREP} -i '(DLL Name:|^'"${REGEX_WS}"'*NEEDED)' | awk '{print $NF}' | sort | uniq | filter_away_system_DLLs`" \ && [ -n "$ODOUT" ] || continue for F in $ODOUT ; do - if [ -n "$DESTDIR" -a -d "${DESTDIR}" ] ; then - OUT="`find \"$DESTDIR\" -type f -name \"$F\" \! -size 0 2>/dev/null | head -1`" \ + if [ -n "$DESTDIR_FILES" ] ; then + OUT="`echo \"$DESTDIR_FILES\" | first_file_named \"$F\"`" \ && [ -n "$OUT" ] && { echo "$OUT" ; SEEN="`expr $SEEN + 1`" ; continue ; } fi - if [ -n "$SEARCH_INPUT_PATH" ] ; then - SEEN_INPUT=false - for D in ${SEARCH_INPUT_PATH_MULTILINE} ; do - OUT="`find \"$D\" -type f -name \"$F\" \! -size 0 2>/dev/null | head -1`" \ - && [ -n "$OUT" ] && { echo "$OUT" ; SEEN_INPUT=true ; break ; } - done - $SEEN_INPUT && { SEEN="`expr $SEEN + 1`" ; continue ; } + if [ -n "$SEARCH_INPUT_FILES" ] ; then + OUT="`echo \"$SEARCH_INPUT_FILES\" | first_file_named \"$F\"`" \ + && [ -n "$OUT" ] && { echo "$OUT" ; SEEN="`expr $SEEN + 1`" ; continue ; } fi if [ -n "$ARCH" -a -d "/usr/${ARCH}" ] ; then OUT="`ls -1 \"/usr/${ARCH}/bin/$F\" \"/usr/${ARCH}/lib/$F\" 2>/dev/null || true`" \ @@ -201,10 +229,10 @@ dllldd_with_tools() ( ) dllldd_with_strings() ( - strings "$@" | tr ' ' '\n' | tr '&' '\n' \ + strings ${STRINGS_DATA_ONLY} "$@" | tr ' ' '\n' | tr '&' '\n' \ | ${EGREP} -i '..*'"${DLLEXT_REGEX_EOL}" | sort | uniq \ | filter_away_system_DLLs \ - | ${EGREP} -vi '^(lib)*%s'"${DLLEXT_REGEX_EOL}" \ + | ${EGREP} -vi '^(.*[/\\])?(lib)*%s'"${DLLEXT_REGEX_EOL}" \ | while read DLL ; do ( # Skip out if we already reported this file # (The for/case loop below is surprisingly expensive on MSYS2) diff --git a/tests/NIT/nit.sh b/tests/NIT/nit.sh index 738586886a..c1095f6c3d 100755 --- a/tests/NIT/nit.sh +++ b/tests/NIT/nit.sh @@ -1235,7 +1235,7 @@ if [ x"${DO_USE_NIT_TESTCERT_CACHE-}" = xyes ] ; then log_info "Found cached NIT certificates in ${CI_CACHE_NIT_HASHDIR}" mkdir -p "${TESTCERT_PATH_BASE}" # See comments above about no TESTCERT_PATH_SEP for shell globs. - cp -prf "${CI_CACHE_NIT_HASHDIR}"/* "${TESTCERT_PATH_BASE}${TESTCERT_PATH_SEP}" || \\ + cp -prf "${CI_CACHE_NIT_HASHDIR}"/* "${TESTCERT_PATH_BASE}${TESTCERT_PATH_SEP}" || \ cp -prfL "${CI_CACHE_NIT_HASHDIR}"/* "${TESTCERT_PATH_BASE}${TESTCERT_PATH_SEP}" # If there is a setup script there, source it to get variables @@ -1252,7 +1252,9 @@ if [ x"${DO_USE_NIT_TESTCERT_CACHE-}" = xyes ] ; then log_warn "FAILED check_NIT_certs with cached data, will generate anew. Removing:" find "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" -ls || true rm -rf "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" || true - mkdir -p "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" + # Do not re-create the (now empty) cache dir: it is populated + # below only if absent, after we generate the certificates + mkdir -p "${TESTCERT_PATH_BASE}" fi else log_info "Did not find a CI_CACHE_NIT_HASHDIR, will populate a new one after generating certificates as '${CI_CACHE_NIT_HASHDIR}'" @@ -1362,7 +1364,8 @@ case "${WITH_SSL_CLIENT}${WITH_SSL_SERVER}" in log_warn "FAILED check_NIT_certs with cached data, will generate anew" rm -rf "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" || true - mkdir -p "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" + # Do not re-create the cache dir, see above + mkdir -p "${TESTCERT_PATH_BASE}" fi # if [ -n "${BASH_VERSION-}" ]; then